Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
83d5197
·
verified ·
1 Parent(s): 01b4a7e

EPFL ALiBi and training provenance 101-150 of 1184

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +5 -0
  2. epfl_final/wandb/run-20260430_021315-cqkg8w61/files/config.yaml +131 -0
  3. epfl_final/wandb/run-20260430_021315-cqkg8w61/files/output.log +1122 -0
  4. epfl_final/wandb/run-20260430_021315-cqkg8w61/files/requirements.txt +97 -0
  5. epfl_final/wandb/run-20260430_021315-cqkg8w61/files/wandb-metadata.json +36 -0
  6. epfl_final/wandb/run-20260430_021315-cqkg8w61/files/wandb-summary.json +1 -0
  7. epfl_final/wandb/run-20260430_022845-fnhn8tt1/files/output.log +469 -0
  8. epfl_final/wandb/run-20260430_022845-fnhn8tt1/files/requirements.txt +133 -0
  9. epfl_final/wandb/run-20260430_022845-fnhn8tt1/files/wandb-metadata.json +36 -0
  10. epfl_final/wandb/run-20260430_022845-fnhn8tt1/logs/debug-internal.log +0 -0
  11. epfl_final/wandb/run-20260430_022845-fnhn8tt1/logs/debug.log +19 -0
  12. epfl_final/wandb/run-20260430_022845-fnhn8tt1/run-fnhn8tt1.wandb +3 -0
  13. epfl_final/wandb/run-20260430_023222-0isb4snr/files/config.yaml +127 -0
  14. epfl_final/wandb/run-20260430_023222-0isb4snr/files/output.log +51 -0
  15. epfl_final/wandb/run-20260430_023222-0isb4snr/files/requirements.txt +97 -0
  16. epfl_final/wandb/run-20260430_023222-0isb4snr/files/wandb-metadata.json +36 -0
  17. epfl_final/wandb/run-20260430_023222-0isb4snr/files/wandb-summary.json +1 -0
  18. epfl_final/wandb/run-20260430_023222-0isb4snr/logs/debug-internal.log +18 -0
  19. epfl_final/wandb/run-20260430_023222-0isb4snr/logs/debug.log +21 -0
  20. epfl_final/wandb/run-20260430_023222-0isb4snr/run-0isb4snr.wandb +0 -0
  21. epfl_final/wandb/run-20260430_153400-3fhhow4v/files/config.yaml +131 -0
  22. epfl_final/wandb/run-20260430_153400-3fhhow4v/files/output.log +611 -0
  23. epfl_final/wandb/run-20260430_153400-3fhhow4v/files/requirements.txt +133 -0
  24. epfl_final/wandb/run-20260430_153400-3fhhow4v/files/wandb-metadata.json +36 -0
  25. epfl_final/wandb/run-20260430_153400-3fhhow4v/files/wandb-summary.json +1 -0
  26. epfl_final/wandb/run-20260430_153400-3fhhow4v/logs/debug-internal.log +0 -0
  27. epfl_final/wandb/run-20260430_153400-3fhhow4v/logs/debug.log +24 -0
  28. epfl_final/wandb/run-20260430_153400-3fhhow4v/run-3fhhow4v.wandb +3 -0
  29. epfl_final/wandb/run-20260430_154427-zytgzzs7/files/config.yaml +131 -0
  30. epfl_final/wandb/run-20260430_154427-zytgzzs7/files/output.log +598 -0
  31. epfl_final/wandb/run-20260430_154427-zytgzzs7/files/requirements.txt +97 -0
  32. epfl_final/wandb/run-20260430_154427-zytgzzs7/files/wandb-metadata.json +36 -0
  33. epfl_final/wandb/run-20260430_154427-zytgzzs7/files/wandb-summary.json +1 -0
  34. epfl_final/wandb/run-20260430_154427-zytgzzs7/logs/debug-internal.log +0 -0
  35. epfl_final/wandb/run-20260430_154427-zytgzzs7/logs/debug.log +24 -0
  36. epfl_final/wandb/run-20260430_154427-zytgzzs7/run-zytgzzs7.wandb +3 -0
  37. epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/config.yaml +130 -0
  38. epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/output.log +195 -0
  39. epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/requirements.txt +97 -0
  40. epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/wandb-metadata.json +36 -0
  41. epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/wandb-summary.json +1 -0
  42. epfl_final/wandb/run-20260430_154531-8xz0mg7d/logs/debug-internal.log +462 -0
  43. epfl_final/wandb/run-20260430_154531-8xz0mg7d/logs/debug.log +21 -0
  44. epfl_final/wandb/run-20260430_154531-8xz0mg7d/run-8xz0mg7d.wandb +3 -0
  45. epfl_final/wandb/run-20260430_154759-yujhhjfq/files/output.log +195 -0
  46. epfl_final/wandb/run-20260430_154759-yujhhjfq/files/requirements.txt +97 -0
  47. epfl_final/wandb/run-20260430_154759-yujhhjfq/files/wandb-metadata.json +36 -0
  48. epfl_final/wandb/run-20260430_154759-yujhhjfq/files/wandb-summary.json +1 -0
  49. epfl_final/wandb/run-20260430_154759-yujhhjfq/logs/debug-internal.log +434 -0
  50. epfl_final/wandb/run-20260430_154759-yujhhjfq/logs/debug.log +21 -0
.gitattributes CHANGED
@@ -206,3 +206,8 @@ epfl_final/SparseAtt/wandb/offline-run-20260921_205453-ghfbqjah/run-ghfbqjah.wan
206
  epfl_final/wandb/run-20260430_020702-cxasagq2/run-cxasagq2.wandb filter=lfs diff=lfs merge=lfs -text
207
  epfl_final/wandb/run-20260430_021108-ngpqwy44/run-ngpqwy44.wandb filter=lfs diff=lfs merge=lfs -text
208
  epfl_final/wandb/run-20260430_021315-cqkg8w61/run-cqkg8w61.wandb filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
206
  epfl_final/wandb/run-20260430_020702-cxasagq2/run-cxasagq2.wandb filter=lfs diff=lfs merge=lfs -text
207
  epfl_final/wandb/run-20260430_021108-ngpqwy44/run-ngpqwy44.wandb filter=lfs diff=lfs merge=lfs -text
208
  epfl_final/wandb/run-20260430_021315-cqkg8w61/run-cqkg8w61.wandb filter=lfs diff=lfs merge=lfs -text
209
+ epfl_final/wandb/run-20260430_022845-fnhn8tt1/run-fnhn8tt1.wandb filter=lfs diff=lfs merge=lfs -text
210
+ epfl_final/wandb/run-20260430_153400-3fhhow4v/run-3fhhow4v.wandb filter=lfs diff=lfs merge=lfs -text
211
+ epfl_final/wandb/run-20260430_154427-zytgzzs7/run-zytgzzs7.wandb filter=lfs diff=lfs merge=lfs -text
212
+ epfl_final/wandb/run-20260430_154531-8xz0mg7d/run-8xz0mg7d.wandb filter=lfs diff=lfs merge=lfs -text
213
+ epfl_final/wandb/run-20260430_154759-yujhhjfq/run-yujhhjfq.wandb filter=lfs diff=lfs merge=lfs -text
epfl_final/wandb/run-20260430_021315-cqkg8w61/files/config.yaml ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ w6c4b8mmqpwfn5x8nr7561y9vmdxch67:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9371709865984"
14
+ used: "4381362393088"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /usr/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-98029aa4-c589-b65b-3ab7-6495256f31ed
25
+ host: h100-workspace-6-0-1
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-04-30T02:13:15.963087Z"
33
+ writerId: w6c4b8mmqpwfn5x8nr7561y9vmdxch67
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 49
40
+ - 51
41
+ "2":
42
+ - 1
43
+ - 11
44
+ - 49
45
+ - 51
46
+ - 71
47
+ "3":
48
+ - 2
49
+ - 15
50
+ - 16
51
+ - 61
52
+ "4": 3.10.12
53
+ "5": 0.26.1
54
+ "12": 0.26.1
55
+ "13": linux-x86_64
56
+ attention_type:
57
+ value: sparse_stochastic
58
+ checkpoint_activations:
59
+ value: false
60
+ eval_batches:
61
+ value: 500
62
+ eval_steps:
63
+ value: 250
64
+ flash_attention:
65
+ value: true
66
+ gradient_accumulation_steps:
67
+ value: 8
68
+ init_from_scratch:
69
+ value: true
70
+ learning_rate:
71
+ value: 0.0005
72
+ logging_steps:
73
+ value: 10
74
+ lr_scheduler:
75
+ value: cosine
76
+ max_grad_norm:
77
+ value: 1
78
+ max_seq_length:
79
+ value: 2048
80
+ max_steps:
81
+ value: 10001
82
+ mixed_precision:
83
+ value: bf16
84
+ model_dtype:
85
+ value: bfloat16
86
+ model_path:
87
+ value: /mloscratch/homes/navasard/qwen3_06b_base
88
+ num_epochs:
89
+ value: 1
90
+ num_workers:
91
+ value: 8
92
+ output_dir:
93
+ value: ./checkpoints/run_20260430_021312
94
+ per_device_batch_size:
95
+ value: 8
96
+ resume_from_checkpoint:
97
+ value: null
98
+ save_steps:
99
+ value: 2500
100
+ seed:
101
+ value: 42
102
+ sparse_block_size:
103
+ value: 128
104
+ sparse_mask_pool_size:
105
+ value: 1000
106
+ sparse_num_random_blocks:
107
+ value: 3
108
+ sparse_val_dense_compare:
109
+ value: false
110
+ strategy:
111
+ value: ddp
112
+ train_path:
113
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
114
+ val_path:
115
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
116
+ verbose:
117
+ value: true
118
+ wandb_enabled:
119
+ value: true
120
+ wandb_name:
121
+ value: null
122
+ wandb_project:
123
+ value: stochastic_sparsity
124
+ wandb_tags:
125
+ value:
126
+ - qwen3
127
+ - 0.6b
128
+ warmup_steps:
129
+ value: 100
130
+ weight_decay:
131
+ value: 0.01
epfl_final/wandb/run-20260430_021315-cqkg8w61/files/output.log ADDED
@@ -0,0 +1,1122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
2
+ Initializing model from scratch (random weights)...
3
+ Checkpoint activations: False
4
+
5
+
6
+ ================================================================================
7
+ MODEL ARCHITECTURE
8
+ ================================================================================
9
+ Model config:
10
+ vocab_size: 151936
11
+ hidden_size: 1024
12
+ num_hidden_layers: 28
13
+ num_attention_heads: 16
14
+ num_key_value_heads: 8
15
+ head_dim: 128
16
+ intermediate_size: 3072
17
+ max_position_embeddings: 32768
18
+ rope_theta: 1000000
19
+ attention_type: sparse_stochastic
20
+ sparse_block_size: 128
21
+ sparse_num_random_blocks: 3
22
+ sparse_mask_pool_size: 1000
23
+ sparse_max_seq_length: 2048
24
+
25
+ Parameter count:
26
+ Total parameters: 596,049,920
27
+ Trainable parameters: 596,049,920
28
+ Non-trainable parameters: 0
29
+ Model size (BF16): 1.19 GB
30
+
31
+ --------------------------------------------------------------------------------
32
+ LAYER-BY-LAYER SUMMARY
33
+ --------------------------------------------------------------------------------
34
+ /home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
35
+
36
+ SOLUTION: Use torch.compile(flex_attention)(...)
37
+
38
+ If you want to debug your score_mod/mask_mod, you can set:
39
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
40
+
41
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
42
+ _warn_once(
43
+ ================================================================================================================================================================
44
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
45
+ ================================================================================================================================================================
46
+ Qwen3ForCausalLM [8, 2048] -- -- True
47
+ ├─Qwen3Model: 1-1 -- [8, 2048, 1024] -- True
48
+ │ └─Embedding: 2-1 [8, 2048] [8, 2048, 1024] 155,582,464 True
49
+ │ └─ModuleList: 2-2 -- -- -- True
50
+ │ │ └─Qwen3DecoderLayer: 3-1 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-2 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-3 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-4 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-5 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-6 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-7 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-8 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-9 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-10 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-11 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-12 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-13 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-14 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-15 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-16 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-17 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-18 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-19 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-20 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-21 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-22 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-23 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-24 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-25 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
75
+ │ │ └─Qwen3DecoderLayer: 3-26 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
76
+ │ │ └─Qwen3DecoderLayer: 3-27 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
77
+ │ │ └─Qwen3DecoderLayer: 3-28 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
78
+ │ └─Qwen3RMSNorm: 2-3 [8, 2048, 1024] [8, 2048, 1024] 1,024 True
79
+ ├─Linear: 1-2 [8, 2048, 1024] [8, 2048, 151936] 155,582,464 True
80
+ ================================================================================================================================================================
81
+ Total params: 751,632,384
82
+ Trainable params: 751,632,384
83
+ Non-trainable params: 0
84
+ Total mult-adds (G): 6.01
85
+ ================================================================================================================================================================
86
+ Input size (MB): 0.13
87
+ Forward/backward pass size (MB): 42035.31
88
+ Params size (MB): 1503.26
89
+ Estimated Total Size (MB): 43538.71
90
+ ================================================================================================================================================================
91
+ ================================================================================
92
+
93
+ Skipping flash-attention hook for sparse_stochastic mode
94
+ Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
95
+ Using pre-tokenized dataset (no runtime tokenization)
96
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
97
+ Loaded 781148 pre-tokenized examples
98
+ Dataset split: 624918 train, 156230 validation
99
+ Training examples: 624918
100
+ Validation examples: 156230
101
+ Steps per epoch: 9764
102
+ Total training steps: 10001
103
+ Starting training...
104
+ Step 10/10001 | Loss: 11.9065 | LR: 5.00e-05 | 7354.1ms/step | 8.7 samples/s
105
+ Step 20/10001 | Loss: 10.6608 | LR: 1.00e-04 | 1308.7ms/step | 48.9 samples/s
106
+ Step 30/10001 | Loss: 9.7102 | LR: 1.50e-04 | 1302.8ms/step | 49.1 samples/s
107
+ Step 40/10001 | Loss: 8.5587 | LR: 2.00e-04 | 1300.2ms/step | 49.2 samples/s
108
+ Step 50/10001 | Loss: 7.9923 | LR: 2.50e-04 | 1302.9ms/step | 49.1 samples/s
109
+ Step 60/10001 | Loss: 7.8980 | LR: 3.00e-04 | 1310.7ms/step | 48.8 samples/s
110
+ Step 70/10001 | Loss: 7.6616 | LR: 3.50e-04 | 1317.9ms/step | 48.6 samples/s
111
+ Step 80/10001 | Loss: 7.4690 | LR: 4.00e-04 | 1320.9ms/step | 48.5 samples/s
112
+ Step 90/10001 | Loss: 7.3970 | LR: 4.50e-04 | 1320.0ms/step | 48.5 samples/s
113
+ Step 100/10001 | Loss: 7.1614 | LR: 5.00e-04 | 1321.2ms/step | 48.4 samples/s
114
+ Step 110/10001 | Loss: 7.0461 | LR: 5.00e-04 | 1323.6ms/step | 48.4 samples/s
115
+ Step 120/10001 | Loss: 6.8752 | LR: 5.00e-04 | 1322.3ms/step | 48.4 samples/s
116
+ Step 130/10001 | Loss: 6.9187 | LR: 5.00e-04 | 1321.6ms/step | 48.4 samples/s
117
+ Step 140/10001 | Loss: 6.7501 | LR: 5.00e-04 | 1322.7ms/step | 48.4 samples/s
118
+ Step 150/10001 | Loss: 6.6482 | LR: 5.00e-04 | 1323.1ms/step | 48.4 samples/s
119
+ Step 160/10001 | Loss: 6.6481 | LR: 5.00e-04 | 1323.8ms/step | 48.3 samples/s
120
+ Step 170/10001 | Loss: 6.5305 | LR: 5.00e-04 | 1326.6ms/step | 48.2 samples/s
121
+ Step 180/10001 | Loss: 6.4572 | LR: 5.00e-04 | 1324.0ms/step | 48.3 samples/s
122
+ Step 190/10001 | Loss: 6.3736 | LR: 5.00e-04 | 1324.5ms/step | 48.3 samples/s
123
+ Step 200/10001 | Loss: 6.3847 | LR: 5.00e-04 | 1325.8ms/step | 48.3 samples/s
124
+ Step 210/10001 | Loss: 6.3007 | LR: 5.00e-04 | 1324.5ms/step | 48.3 samples/s
125
+ Step 220/10001 | Loss: 6.2210 | LR: 5.00e-04 | 1324.2ms/step | 48.3 samples/s
126
+ Step 230/10001 | Loss: 6.1995 | LR: 5.00e-04 | 1326.4ms/step | 48.2 samples/s
127
+ Step 240/10001 | Loss: 6.1380 | LR: 5.00e-04 | 1327.0ms/step | 48.2 samples/s
128
+ Step 250/10001 | Loss: 6.0570 | LR: 5.00e-04 | 1326.9ms/step | 48.2 samples/s
129
+ Validation at step 250: Loss = 6.0316, Perplexity = 416.38
130
+ Step 260/10001 | Loss: 6.0655 | LR: 5.00e-04 | 29221.5ms/step | 2.2 samples/s
131
+ Step 270/10001 | Loss: 5.9812 | LR: 5.00e-04 | 1331.8ms/step | 48.1 samples/s
132
+ Step 280/10001 | Loss: 5.8995 | LR: 5.00e-04 | 1330.6ms/step | 48.1 samples/s
133
+ Step 290/10001 | Loss: 5.8888 | LR: 5.00e-04 | 1333.2ms/step | 48.0 samples/s
134
+ Step 300/10001 | Loss: 5.8452 | LR: 4.99e-04 | 1334.5ms/step | 48.0 samples/s
135
+ Step 310/10001 | Loss: 5.8017 | LR: 4.99e-04 | 1331.8ms/step | 48.1 samples/s
136
+ Step 320/10001 | Loss: 5.7402 | LR: 4.99e-04 | 1331.6ms/step | 48.1 samples/s
137
+ Step 330/10001 | Loss: 5.7520 | LR: 4.99e-04 | 1331.2ms/step | 48.1 samples/s
138
+ Step 340/10001 | Loss: 5.6775 | LR: 4.99e-04 | 1331.1ms/step | 48.1 samples/s
139
+ Step 350/10001 | Loss: 5.6606 | LR: 4.99e-04 | 1328.2ms/step | 48.2 samples/s
140
+ Step 360/10001 | Loss: 5.5800 | LR: 4.99e-04 | 1328.5ms/step | 48.2 samples/s
141
+ Step 370/10001 | Loss: 5.5788 | LR: 4.99e-04 | 1422.8ms/step | 45.0 samples/s
142
+ Step 380/10001 | Loss: 5.5431 | LR: 4.99e-04 | 1332.9ms/step | 48.0 samples/s
143
+ Step 390/10001 | Loss: 5.4958 | LR: 4.99e-04 | 1329.8ms/step | 48.1 samples/s
144
+ Step 400/10001 | Loss: 5.4641 | LR: 4.99e-04 | 1331.2ms/step | 48.1 samples/s
145
+ Step 410/10001 | Loss: 5.4242 | LR: 4.99e-04 | 1329.8ms/step | 48.1 samples/s
146
+ Step 420/10001 | Loss: 5.3805 | LR: 4.99e-04 | 1329.5ms/step | 48.1 samples/s
147
+ Step 430/10001 | Loss: 5.3513 | LR: 4.99e-04 | 1329.5ms/step | 48.1 samples/s
148
+ Step 440/10001 | Loss: 5.3323 | LR: 4.99e-04 | 1327.7ms/step | 48.2 samples/s
149
+ Step 450/10001 | Loss: 5.2954 | LR: 4.98e-04 | 1327.8ms/step | 48.2 samples/s
150
+ Step 460/10001 | Loss: 5.2529 | LR: 4.98e-04 | 1328.7ms/step | 48.2 samples/s
151
+ Step 470/10001 | Loss: 5.2398 | LR: 4.98e-04 | 1330.3ms/step | 48.1 samples/s
152
+ Step 480/10001 | Loss: 5.1961 | LR: 4.98e-04 | 1333.7ms/step | 48.0 samples/s
153
+ Step 490/10001 | Loss: 5.1872 | LR: 4.98e-04 | 1331.4ms/step | 48.1 samples/s
154
+ Step 500/10001 | Loss: 5.1267 | LR: 4.98e-04 | 1332.3ms/step | 48.0 samples/s
155
+ Validation at step 500: Loss = 5.1277, Perplexity = 168.63
156
+ Step 510/10001 | Loss: 5.1085 | LR: 4.98e-04 | 26758.3ms/step | 2.4 samples/s
157
+ Step 520/10001 | Loss: 5.0939 | LR: 4.98e-04 | 1330.3ms/step | 48.1 samples/s
158
+ Step 530/10001 | Loss: 5.0641 | LR: 4.98e-04 | 1329.0ms/step | 48.2 samples/s
159
+ Step 540/10001 | Loss: 5.0253 | LR: 4.98e-04 | 1329.3ms/step | 48.1 samples/s
160
+ Step 550/10001 | Loss: 4.9978 | LR: 4.97e-04 | 1332.6ms/step | 48.0 samples/s
161
+ Step 560/10001 | Loss: 5.0026 | LR: 4.97e-04 | 1333.2ms/step | 48.0 samples/s
162
+ Step 570/10001 | Loss: 4.9575 | LR: 4.97e-04 | 1334.7ms/step | 48.0 samples/s
163
+ Step 580/10001 | Loss: 4.9475 | LR: 4.97e-04 | 1333.0ms/step | 48.0 samples/s
164
+ Step 590/10001 | Loss: 4.9189 | LR: 4.97e-04 | 1331.5ms/step | 48.1 samples/s
165
+ Step 600/10001 | Loss: 4.9125 | LR: 4.97e-04 | 1330.9ms/step | 48.1 samples/s
166
+ Step 610/10001 | Loss: 4.8883 | LR: 4.97e-04 | 1334.0ms/step | 48.0 samples/s
167
+ Step 620/10001 | Loss: 4.8556 | LR: 4.97e-04 | 1330.4ms/step | 48.1 samples/s
168
+ Step 630/10001 | Loss: 4.8339 | LR: 4.96e-04 | 1330.0ms/step | 48.1 samples/s
169
+ Step 640/10001 | Loss: 4.7984 | LR: 4.96e-04 | 1331.5ms/step | 48.1 samples/s
170
+ Step 650/10001 | Loss: 4.7711 | LR: 4.96e-04 | 1331.7ms/step | 48.1 samples/s
171
+ Step 660/10001 | Loss: 4.7930 | LR: 4.96e-04 | 1332.9ms/step | 48.0 samples/s
172
+ Step 670/10001 | Loss: 4.7488 | LR: 4.96e-04 | 1330.1ms/step | 48.1 samples/s
173
+ Step 680/10001 | Loss: 4.7305 | LR: 4.96e-04 | 1327.9ms/step | 48.2 samples/s
174
+ Step 690/10001 | Loss: 4.7155 | LR: 4.96e-04 | 1327.9ms/step | 48.2 samples/s
175
+ Step 700/10001 | Loss: 4.7298 | LR: 4.95e-04 | 1329.9ms/step | 48.1 samples/s
176
+ Step 710/10001 | Loss: 4.6989 | LR: 4.95e-04 | 1329.7ms/step | 48.1 samples/s
177
+ Step 720/10001 | Loss: 4.6972 | LR: 4.95e-04 | 1328.0ms/step | 48.2 samples/s
178
+ Step 730/10001 | Loss: 4.6770 | LR: 4.95e-04 | 1328.5ms/step | 48.2 samples/s
179
+ Step 740/10001 | Loss: 4.6602 | LR: 4.95e-04 | 1329.8ms/step | 48.1 samples/s
180
+ Step 750/10001 | Loss: 4.6449 | LR: 4.95e-04 | 1329.0ms/step | 48.2 samples/s
181
+ Validation at step 750: Loss = 4.6275, Perplexity = 102.26
182
+ Step 760/10001 | Loss: 4.5983 | LR: 4.95e-04 | 26749.6ms/step | 2.4 samples/s
183
+ Step 770/10001 | Loss: 4.6315 | LR: 4.94e-04 | 1330.7ms/step | 48.1 samples/s
184
+ Step 780/10001 | Loss: 4.5755 | LR: 4.94e-04 | 1331.7ms/step | 48.1 samples/s
185
+ Step 790/10001 | Loss: 4.5795 | LR: 4.94e-04 | 1331.7ms/step | 48.1 samples/s
186
+ Step 800/10001 | Loss: 4.5798 | LR: 4.94e-04 | 1330.8ms/step | 48.1 samples/s
187
+ Step 810/10001 | Loss: 4.5558 | LR: 4.94e-04 | 1329.9ms/step | 48.1 samples/s
188
+ Step 820/10001 | Loss: 4.5201 | LR: 4.94e-04 | 1329.0ms/step | 48.2 samples/s
189
+ Step 830/10001 | Loss: 4.5349 | LR: 4.93e-04 | 1328.4ms/step | 48.2 samples/s
190
+ Step 840/10001 | Loss: 4.5028 | LR: 4.93e-04 | 1331.5ms/step | 48.1 samples/s
191
+ Step 850/10001 | Loss: 4.5186 | LR: 4.93e-04 | 1329.8ms/step | 48.1 samples/s
192
+ Step 860/10001 | Loss: 4.4797 | LR: 4.93e-04 | 1330.6ms/step | 48.1 samples/s
193
+ Step 870/10001 | Loss: 4.4630 | LR: 4.93e-04 | 1331.4ms/step | 48.1 samples/s
194
+ Step 880/10001 | Loss: 4.4730 | LR: 4.92e-04 | 1333.3ms/step | 48.0 samples/s
195
+ Step 890/10001 | Loss: 4.4811 | LR: 4.92e-04 | 1331.9ms/step | 48.1 samples/s
196
+ Step 900/10001 | Loss: 4.4364 | LR: 4.92e-04 | 1333.1ms/step | 48.0 samples/s
197
+ Step 910/10001 | Loss: 4.4544 | LR: 4.92e-04 | 1329.4ms/step | 48.1 samples/s
198
+ Step 920/10001 | Loss: 4.4056 | LR: 4.92e-04 | 1328.3ms/step | 48.2 samples/s
199
+ Step 930/10001 | Loss: 4.4156 | LR: 4.91e-04 | 1328.9ms/step | 48.2 samples/s
200
+ Step 940/10001 | Loss: 4.4184 | LR: 4.91e-04 | 1330.4ms/step | 48.1 samples/s
201
+ Step 950/10001 | Loss: 4.3873 | LR: 4.91e-04 | 1330.3ms/step | 48.1 samples/s
202
+ Step 960/10001 | Loss: 4.3539 | LR: 4.91e-04 | 1331.6ms/step | 48.1 samples/s
203
+ Step 970/10001 | Loss: 4.3688 | LR: 4.91e-04 | 1330.6ms/step | 48.1 samples/s
204
+ Step 980/10001 | Loss: 4.3324 | LR: 4.90e-04 | 1330.2ms/step | 48.1 samples/s
205
+ Step 990/10001 | Loss: 4.3630 | LR: 4.90e-04 | 1331.4ms/step | 48.1 samples/s
206
+ Step 1000/10001 | Loss: 4.3634 | LR: 4.90e-04 | 1330.6ms/step | 48.1 samples/s
207
+ Validation at step 1000: Loss = 4.3516, Perplexity = 77.60
208
+ Step 1010/10001 | Loss: 4.3262 | LR: 4.90e-04 | 26761.3ms/step | 2.4 samples/s
209
+ Step 1020/10001 | Loss: 4.3454 | LR: 4.89e-04 | 1329.0ms/step | 48.2 samples/s
210
+ Step 1030/10001 | Loss: 4.3136 | LR: 4.89e-04 | 1330.7ms/step | 48.1 samples/s
211
+ Step 1040/10001 | Loss: 4.2971 | LR: 4.89e-04 | 1329.2ms/step | 48.1 samples/s
212
+ Step 1050/10001 | Loss: 4.3149 | LR: 4.89e-04 | 1328.7ms/step | 48.2 samples/s
213
+ Step 1060/10001 | Loss: 4.3192 | LR: 4.88e-04 | 1329.9ms/step | 48.1 samples/s
214
+ Step 1070/10001 | Loss: 4.3268 | LR: 4.88e-04 | 1329.2ms/step | 48.2 samples/s
215
+ Step 1080/10001 | Loss: 4.2723 | LR: 4.88e-04 | 1328.8ms/step | 48.2 samples/s
216
+ Step 1090/10001 | Loss: 4.2753 | LR: 4.88e-04 | 1329.1ms/step | 48.2 samples/s
217
+ Step 1100/10001 | Loss: 4.2662 | LR: 4.88e-04 | 1411.0ms/step | 45.4 samples/s
218
+ Step 1110/10001 | Loss: 4.2789 | LR: 4.87e-04 | 1328.1ms/step | 48.2 samples/s
219
+ Step 1120/10001 | Loss: 4.2512 | LR: 4.87e-04 | 1330.5ms/step | 48.1 samples/s
220
+ Step 1130/10001 | Loss: 4.2704 | LR: 4.87e-04 | 1329.7ms/step | 48.1 samples/s
221
+ Step 1140/10001 | Loss: 4.2665 | LR: 4.87e-04 | 1330.4ms/step | 48.1 samples/s
222
+ Step 1150/10001 | Loss: 4.2354 | LR: 4.86e-04 | 1330.2ms/step | 48.1 samples/s
223
+ Step 1160/10001 | Loss: 4.2335 | LR: 4.86e-04 | 1329.4ms/step | 48.1 samples/s
224
+ Step 1170/10001 | Loss: 4.2088 | LR: 4.86e-04 | 1329.4ms/step | 48.1 samples/s
225
+ Step 1180/10001 | Loss: 4.2056 | LR: 4.85e-04 | 1329.2ms/step | 48.1 samples/s
226
+ Step 1190/10001 | Loss: 4.2316 | LR: 4.85e-04 | 1330.1ms/step | 48.1 samples/s
227
+ Step 1200/10001 | Loss: 4.2042 | LR: 4.85e-04 | 1329.0ms/step | 48.2 samples/s
228
+ Step 1210/10001 | Loss: 4.2401 | LR: 4.85e-04 | 1327.1ms/step | 48.2 samples/s
229
+ Step 1220/10001 | Loss: 4.2302 | LR: 4.84e-04 | 1328.2ms/step | 48.2 samples/s
230
+ Step 1230/10001 | Loss: 4.2190 | LR: 4.84e-04 | 1329.7ms/step | 48.1 samples/s
231
+ Step 1240/10001 | Loss: 4.2058 | LR: 4.84e-04 | 1330.1ms/step | 48.1 samples/s
232
+ Step 1250/10001 | Loss: 4.1860 | LR: 4.84e-04 | 1330.0ms/step | 48.1 samples/s
233
+ Validation at step 1250: Loss = 4.1767, Perplexity = 65.15
234
+ Step 1260/10001 | Loss: 4.2101 | LR: 4.83e-04 | 26725.0ms/step | 2.4 samples/s
235
+ Step 1270/10001 | Loss: 4.1616 | LR: 4.83e-04 | 1331.3ms/step | 48.1 samples/s
236
+ Step 1280/10001 | Loss: 4.1503 | LR: 4.83e-04 | 1330.7ms/step | 48.1 samples/s
237
+ Step 1290/10001 | Loss: 4.1691 | LR: 4.82e-04 | 1333.2ms/step | 48.0 samples/s
238
+ Step 1300/10001 | Loss: 4.1650 | LR: 4.82e-04 | 1330.2ms/step | 48.1 samples/s
239
+ Step 1310/10001 | Loss: 4.1602 | LR: 4.82e-04 | 1327.8ms/step | 48.2 samples/s
240
+ Step 1320/10001 | Loss: 4.1648 | LR: 4.82e-04 | 1331.9ms/step | 48.1 samples/s
241
+ Step 1330/10001 | Loss: 4.1598 | LR: 4.81e-04 | 1330.3ms/step | 48.1 samples/s
242
+ Step 1340/10001 | Loss: 4.1621 | LR: 4.81e-04 | 1329.3ms/step | 48.1 samples/s
243
+ Step 1350/10001 | Loss: 4.1284 | LR: 4.81e-04 | 1331.2ms/step | 48.1 samples/s
244
+ Step 1360/10001 | Loss: 4.1299 | LR: 4.80e-04 | 1330.5ms/step | 48.1 samples/s
245
+ Step 1370/10001 | Loss: 4.1078 | LR: 4.80e-04 | 1332.1ms/step | 48.0 samples/s
246
+ Step 1380/10001 | Loss: 4.0988 | LR: 4.80e-04 | 1329.7ms/step | 48.1 samples/s
247
+ Step 1390/10001 | Loss: 4.0995 | LR: 4.79e-04 | 1329.4ms/step | 48.1 samples/s
248
+ Step 1400/10001 | Loss: 4.0584 | LR: 4.79e-04 | 1329.9ms/step | 48.1 samples/s
249
+ Step 1410/10001 | Loss: 4.1076 | LR: 4.79e-04 | 1330.1ms/step | 48.1 samples/s
250
+ Step 1420/10001 | Loss: 4.0885 | LR: 4.78e-04 | 1329.2ms/step | 48.1 samples/s
251
+ Step 1430/10001 | Loss: 4.1030 | LR: 4.78e-04 | 1327.5ms/step | 48.2 samples/s
252
+ Step 1440/10001 | Loss: 4.0835 | LR: 4.78e-04 | 1329.4ms/step | 48.1 samples/s
253
+ Step 1450/10001 | Loss: 4.0876 | LR: 4.77e-04 | 1330.0ms/step | 48.1 samples/s
254
+ Step 1460/10001 | Loss: 4.1027 | LR: 4.77e-04 | 1325.8ms/step | 48.3 samples/s
255
+ Step 1470/10001 | Loss: 4.0836 | LR: 4.77e-04 | 1327.4ms/step | 48.2 samples/s
256
+ Step 1480/10001 | Loss: 4.0360 | LR: 4.76e-04 | 1327.5ms/step | 48.2 samples/s
257
+ Step 1490/10001 | Loss: 4.0701 | LR: 4.76e-04 | 1327.4ms/step | 48.2 samples/s
258
+ Step 1500/10001 | Loss: 4.0542 | LR: 4.76e-04 | 1326.7ms/step | 48.2 samples/s
259
+ Validation at step 1500: Loss = 4.0592, Perplexity = 57.93
260
+ Step 1510/10001 | Loss: 4.0671 | LR: 4.75e-04 | 26730.8ms/step | 2.4 samples/s
261
+ Step 1520/10001 | Loss: 4.0414 | LR: 4.75e-04 | 1332.3ms/step | 48.0 samples/s
262
+ Step 1530/10001 | Loss: 4.0497 | LR: 4.75e-04 | 1330.0ms/step | 48.1 samples/s
263
+ Step 1540/10001 | Loss: 4.0356 | LR: 4.74e-04 | 1330.3ms/step | 48.1 samples/s
264
+ Step 1550/10001 | Loss: 4.0180 | LR: 4.74e-04 | 1332.6ms/step | 48.0 samples/s
265
+ Step 1560/10001 | Loss: 4.0447 | LR: 4.74e-04 | 1329.6ms/step | 48.1 samples/s
266
+ Step 1570/10001 | Loss: 4.0267 | LR: 4.73e-04 | 1331.8ms/step | 48.1 samples/s
267
+ Step 1580/10001 | Loss: 4.0129 | LR: 4.73e-04 | 1330.1ms/step | 48.1 samples/s
268
+ Step 1590/10001 | Loss: 4.0321 | LR: 4.73e-04 | 1330.3ms/step | 48.1 samples/s
269
+ Step 1600/10001 | Loss: 4.0077 | LR: 4.72e-04 | 1328.8ms/step | 48.2 samples/s
270
+ Step 1610/10001 | Loss: 3.9954 | LR: 4.72e-04 | 1329.1ms/step | 48.2 samples/s
271
+ Step 1620/10001 | Loss: 4.0128 | LR: 4.71e-04 | 1328.1ms/step | 48.2 samples/s
272
+ Step 1630/10001 | Loss: 3.9963 | LR: 4.71e-04 | 1329.7ms/step | 48.1 samples/s
273
+ Step 1640/10001 | Loss: 3.9881 | LR: 4.71e-04 | 1327.1ms/step | 48.2 samples/s
274
+ Step 1650/10001 | Loss: 3.9903 | LR: 4.70e-04 | 1328.5ms/step | 48.2 samples/s
275
+ Step 1660/10001 | Loss: 4.0076 | LR: 4.70e-04 | 1328.2ms/step | 48.2 samples/s
276
+ Step 1670/10001 | Loss: 3.9666 | LR: 4.70e-04 | 1327.7ms/step | 48.2 samples/s
277
+ Step 1680/10001 | Loss: 3.9968 | LR: 4.69e-04 | 1328.5ms/step | 48.2 samples/s
278
+ Step 1690/10001 | Loss: 3.9782 | LR: 4.69e-04 | 1327.9ms/step | 48.2 samples/s
279
+ Step 1700/10001 | Loss: 3.9827 | LR: 4.68e-04 | 1327.2ms/step | 48.2 samples/s
280
+ Step 1710/10001 | Loss: 3.9690 | LR: 4.68e-04 | 1328.0ms/step | 48.2 samples/s
281
+ Step 1720/10001 | Loss: 3.9653 | LR: 4.68e-04 | 1329.2ms/step | 48.2 samples/s
282
+ Step 1730/10001 | Loss: 3.9943 | LR: 4.67e-04 | 1326.8ms/step | 48.2 samples/s
283
+ Step 1740/10001 | Loss: 3.9648 | LR: 4.67e-04 | 1329.7ms/step | 48.1 samples/s
284
+ Step 1750/10001 | Loss: 3.9937 | LR: 4.67e-04 | 1329.6ms/step | 48.1 samples/s
285
+ Validation at step 1750: Loss = 3.9696, Perplexity = 52.96
286
+ Step 1760/10001 | Loss: 3.9820 | LR: 4.66e-04 | 26733.3ms/step | 2.4 samples/s
287
+ Step 1770/10001 | Loss: 3.9353 | LR: 4.66e-04 | 1331.2ms/step | 48.1 samples/s
288
+ Step 1780/10001 | Loss: 3.9542 | LR: 4.65e-04 | 1330.9ms/step | 48.1 samples/s
289
+ Step 1790/10001 | Loss: 3.9658 | LR: 4.65e-04 | 1331.8ms/step | 48.1 samples/s
290
+ Step 1800/10001 | Loss: 3.9586 | LR: 4.65e-04 | 1328.3ms/step | 48.2 samples/s
291
+ Step 1810/10001 | Loss: 3.9389 | LR: 4.64e-04 | 1326.7ms/step | 48.2 samples/s
292
+ Step 1820/10001 | Loss: 3.9309 | LR: 4.64e-04 | 1326.4ms/step | 48.3 samples/s
293
+ Step 1830/10001 | Loss: 3.9572 | LR: 4.63e-04 | 1412.0ms/step | 45.3 samples/s
294
+ Step 1840/10001 | Loss: 3.9255 | LR: 4.63e-04 | 1328.8ms/step | 48.2 samples/s
295
+ Step 1850/10001 | Loss: 3.9308 | LR: 4.62e-04 | 1328.6ms/step | 48.2 samples/s
296
+ Step 1860/10001 | Loss: 3.9317 | LR: 4.62e-04 | 1327.9ms/step | 48.2 samples/s
297
+ Step 1870/10001 | Loss: 3.9317 | LR: 4.62e-04 | 1327.2ms/step | 48.2 samples/s
298
+ Step 1880/10001 | Loss: 3.9295 | LR: 4.61e-04 | 1325.7ms/step | 48.3 samples/s
299
+ Step 1890/10001 | Loss: 3.9193 | LR: 4.61e-04 | 1327.7ms/step | 48.2 samples/s
300
+ Step 1900/10001 | Loss: 3.9388 | LR: 4.60e-04 | 1326.0ms/step | 48.3 samples/s
301
+ Step 1910/10001 | Loss: 3.9277 | LR: 4.60e-04 | 1324.7ms/step | 48.3 samples/s
302
+ Step 1920/10001 | Loss: 3.9373 | LR: 4.59e-04 | 1326.2ms/step | 48.3 samples/s
303
+ Step 1930/10001 | Loss: 3.9259 | LR: 4.59e-04 | 1324.0ms/step | 48.3 samples/s
304
+ Step 1940/10001 | Loss: 3.9038 | LR: 4.59e-04 | 1326.2ms/step | 48.3 samples/s
305
+ Step 1950/10001 | Loss: 3.9115 | LR: 4.58e-04 | 1328.1ms/step | 48.2 samples/s
306
+ Step 1960/10001 | Loss: 3.9167 | LR: 4.58e-04 | 1327.6ms/step | 48.2 samples/s
307
+ Step 1970/10001 | Loss: 3.8973 | LR: 4.57e-04 | 1328.2ms/step | 48.2 samples/s
308
+ Step 1980/10001 | Loss: 3.9163 | LR: 4.57e-04 | 1330.6ms/step | 48.1 samples/s
309
+ Step 1990/10001 | Loss: 3.9049 | LR: 4.56e-04 | 1329.5ms/step | 48.1 samples/s
310
+ Step 2000/10001 | Loss: 3.8877 | LR: 4.56e-04 | 1329.3ms/step | 48.1 samples/s
311
+ Validation at step 2000: Loss = 3.8976, Perplexity = 49.28
312
+ Step 2010/10001 | Loss: 3.8937 | LR: 4.55e-04 | 26702.8ms/step | 2.4 samples/s
313
+ Step 2020/10001 | Loss: 3.9185 | LR: 4.55e-04 | 1327.5ms/step | 48.2 samples/s
314
+ Step 2030/10001 | Loss: 3.8831 | LR: 4.55e-04 | 1324.4ms/step | 48.3 samples/s
315
+ Step 2040/10001 | Loss: 3.8690 | LR: 4.54e-04 | 1326.4ms/step | 48.3 samples/s
316
+ Step 2050/10001 | Loss: 3.8933 | LR: 4.54e-04 | 1327.4ms/step | 48.2 samples/s
317
+ Step 2060/10001 | Loss: 3.8869 | LR: 4.53e-04 | 1322.3ms/step | 48.4 samples/s
318
+ Step 2070/10001 | Loss: 3.8682 | LR: 4.53e-04 | 1322.6ms/step | 48.4 samples/s
319
+ Step 2080/10001 | Loss: 3.9098 | LR: 4.52e-04 | 1323.7ms/step | 48.3 samples/s
320
+ Step 2090/10001 | Loss: 3.8658 | LR: 4.52e-04 | 1324.9ms/step | 48.3 samples/s
321
+ Step 2100/10001 | Loss: 3.8705 | LR: 4.51e-04 | 1323.4ms/step | 48.4 samples/s
322
+ Step 2110/10001 | Loss: 3.8517 | LR: 4.51e-04 | 1322.4ms/step | 48.4 samples/s
323
+ Step 2120/10001 | Loss: 3.8747 | LR: 4.50e-04 | 1324.7ms/step | 48.3 samples/s
324
+ Step 2130/10001 | Loss: 3.8530 | LR: 4.50e-04 | 1324.9ms/step | 48.3 samples/s
325
+ Step 2140/10001 | Loss: 3.8719 | LR: 4.49e-04 | 1328.3ms/step | 48.2 samples/s
326
+ Step 2150/10001 | Loss: 3.8522 | LR: 4.49e-04 | 1327.9ms/step | 48.2 samples/s
327
+ Step 2160/10001 | Loss: 3.8484 | LR: 4.48e-04 | 1328.2ms/step | 48.2 samples/s
328
+ Step 2170/10001 | Loss: 3.8674 | LR: 4.48e-04 | 1326.3ms/step | 48.3 samples/s
329
+ Step 2180/10001 | Loss: 3.8667 | LR: 4.48e-04 | 1327.3ms/step | 48.2 samples/s
330
+ Step 2190/10001 | Loss: 3.8558 | LR: 4.47e-04 | 1329.0ms/step | 48.2 samples/s
331
+ Step 2200/10001 | Loss: 3.8640 | LR: 4.47e-04 | 1328.9ms/step | 48.2 samples/s
332
+ Step 2210/10001 | Loss: 3.8177 | LR: 4.46e-04 | 1332.0ms/step | 48.0 samples/s
333
+ Step 2220/10001 | Loss: 3.8592 | LR: 4.46e-04 | 1330.2ms/step | 48.1 samples/s
334
+ Step 2230/10001 | Loss: 3.8641 | LR: 4.45e-04 | 1329.4ms/step | 48.1 samples/s
335
+ Step 2240/10001 | Loss: 3.8340 | LR: 4.45e-04 | 1324.4ms/step | 48.3 samples/s
336
+ Step 2250/10001 | Loss: 3.8003 | LR: 4.44e-04 | 1322.6ms/step | 48.4 samples/s
337
+ Validation at step 2250: Loss = 3.8406, Perplexity = 46.55
338
+ Step 2260/10001 | Loss: 3.8419 | LR: 4.44e-04 | 26705.3ms/step | 2.4 samples/s
339
+ Step 2270/10001 | Loss: 3.8563 | LR: 4.43e-04 | 1332.5ms/step | 48.0 samples/s
340
+ Step 2280/10001 | Loss: 3.8261 | LR: 4.43e-04 | 1329.9ms/step | 48.1 samples/s
341
+ Step 2290/10001 | Loss: 3.8559 | LR: 4.42e-04 | 1329.3ms/step | 48.1 samples/s
342
+ Step 2300/10001 | Loss: 3.8644 | LR: 4.42e-04 | 1330.8ms/step | 48.1 samples/s
343
+ Step 2310/10001 | Loss: 3.8491 | LR: 4.41e-04 | 1330.0ms/step | 48.1 samples/s
344
+ Step 2320/10001 | Loss: 3.8146 | LR: 4.40e-04 | 1330.6ms/step | 48.1 samples/s
345
+ Step 2330/10001 | Loss: 3.8126 | LR: 4.40e-04 | 1334.4ms/step | 48.0 samples/s
346
+ Step 2340/10001 | Loss: 3.8279 | LR: 4.39e-04 | 1336.3ms/step | 47.9 samples/s
347
+ Step 2350/10001 | Loss: 3.8462 | LR: 4.39e-04 | 1332.0ms/step | 48.0 samples/s
348
+ Step 2360/10001 | Loss: 3.8282 | LR: 4.38e-04 | 1329.7ms/step | 48.1 samples/s
349
+ Step 2370/10001 | Loss: 3.8369 | LR: 4.38e-04 | 1328.9ms/step | 48.2 samples/s
350
+ Step 2380/10001 | Loss: 3.8149 | LR: 4.37e-04 | 1331.6ms/step | 48.1 samples/s
351
+ Step 2390/10001 | Loss: 3.7925 | LR: 4.37e-04 | 1329.5ms/step | 48.1 samples/s
352
+ Step 2400/10001 | Loss: 3.8068 | LR: 4.36e-04 | 1329.0ms/step | 48.2 samples/s
353
+ Step 2410/10001 | Loss: 3.8317 | LR: 4.36e-04 | 1332.2ms/step | 48.0 samples/s
354
+ Step 2420/10001 | Loss: 3.8077 | LR: 4.35e-04 | 1329.5ms/step | 48.1 samples/s
355
+ Step 2430/10001 | Loss: 3.7994 | LR: 4.35e-04 | 1330.5ms/step | 48.1 samples/s
356
+ Step 2440/10001 | Loss: 3.7923 | LR: 4.34e-04 | 1332.4ms/step | 48.0 samples/s
357
+ Step 2450/10001 | Loss: 3.8038 | LR: 4.34e-04 | 1330.4ms/step | 48.1 samples/s
358
+ Step 2460/10001 | Loss: 3.7903 | LR: 4.33e-04 | 1329.9ms/step | 48.1 samples/s
359
+ Step 2470/10001 | Loss: 3.8138 | LR: 4.33e-04 | 1331.0ms/step | 48.1 samples/s
360
+ Step 2480/10001 | Loss: 3.8117 | LR: 4.32e-04 | 1329.0ms/step | 48.2 samples/s
361
+ Step 2490/10001 | Loss: 3.7960 | LR: 4.31e-04 | 1330.7ms/step | 48.1 samples/s
362
+ Step 2500/10001 | Loss: 3.7975 | LR: 4.31e-04 | 1332.5ms/step | 48.0 samples/s
363
+ Validation at step 2500: Loss = 3.7921, Perplexity = 44.35
364
+ Saved checkpoint to checkpoints/run_20260430_021312/checkpoint-2500
365
+ Step 2510/10001 | Loss: 3.7811 | LR: 4.30e-04 | 27219.8ms/step | 2.4 samples/s
366
+ Step 2520/10001 | Loss: 3.7620 | LR: 4.30e-04 | 1326.6ms/step | 48.2 samples/s
367
+ Step 2530/10001 | Loss: 3.7856 | LR: 4.29e-04 | 1324.7ms/step | 48.3 samples/s
368
+ Step 2540/10001 | Loss: 3.7864 | LR: 4.29e-04 | 1326.9ms/step | 48.2 samples/s
369
+ Step 2550/10001 | Loss: 3.8044 | LR: 4.28e-04 | 1411.1ms/step | 45.4 samples/s
370
+ Step 2560/10001 | Loss: 3.7431 | LR: 4.28e-04 | 1327.9ms/step | 48.2 samples/s
371
+ Step 2570/10001 | Loss: 3.7783 | LR: 4.27e-04 | 1328.5ms/step | 48.2 samples/s
372
+ Step 2580/10001 | Loss: 3.7698 | LR: 4.27e-04 | 1329.7ms/step | 48.1 samples/s
373
+ Step 2590/10001 | Loss: 3.7819 | LR: 4.26e-04 | 1331.0ms/step | 48.1 samples/s
374
+ Step 2600/10001 | Loss: 3.7581 | LR: 4.25e-04 | 1328.3ms/step | 48.2 samples/s
375
+ Step 2610/10001 | Loss: 3.8014 | LR: 4.25e-04 | 1328.1ms/step | 48.2 samples/s
376
+ Step 2620/10001 | Loss: 3.7590 | LR: 4.24e-04 | 1327.5ms/step | 48.2 samples/s
377
+ Step 2630/10001 | Loss: 3.7515 | LR: 4.24e-04 | 1327.7ms/step | 48.2 samples/s
378
+ Step 2640/10001 | Loss: 3.7567 | LR: 4.23e-04 | 1328.1ms/step | 48.2 samples/s
379
+ Step 2650/10001 | Loss: 3.7375 | LR: 4.23e-04 | 1328.0ms/step | 48.2 samples/s
380
+ Step 2660/10001 | Loss: 3.7694 | LR: 4.22e-04 | 1328.5ms/step | 48.2 samples/s
381
+ Step 2670/10001 | Loss: 3.7943 | LR: 4.21e-04 | 1328.5ms/step | 48.2 samples/s
382
+ Step 2680/10001 | Loss: 3.7576 | LR: 4.21e-04 | 1329.4ms/step | 48.1 samples/s
383
+ Step 2690/10001 | Loss: 3.7364 | LR: 4.20e-04 | 1328.8ms/step | 48.2 samples/s
384
+ Step 2700/10001 | Loss: 3.7559 | LR: 4.20e-04 | 1329.3ms/step | 48.1 samples/s
385
+ Step 2710/10001 | Loss: 3.7476 | LR: 4.19e-04 | 1328.3ms/step | 48.2 samples/s
386
+ Step 2720/10001 | Loss: 3.7305 | LR: 4.18e-04 | 1328.2ms/step | 48.2 samples/s
387
+ Step 2730/10001 | Loss: 3.7411 | LR: 4.18e-04 | 1328.6ms/step | 48.2 samples/s
388
+ Step 2740/10001 | Loss: 3.7542 | LR: 4.17e-04 | 1327.3ms/step | 48.2 samples/s
389
+ Step 2750/10001 | Loss: 3.7656 | LR: 4.17e-04 | 1328.7ms/step | 48.2 samples/s
390
+ Validation at step 2750: Loss = 3.7498, Perplexity = 42.51
391
+ Step 2760/10001 | Loss: 3.7596 | LR: 4.16e-04 | 26727.8ms/step | 2.4 samples/s
392
+ Step 2770/10001 | Loss: 3.7666 | LR: 4.16e-04 | 1330.1ms/step | 48.1 samples/s
393
+ Step 2780/10001 | Loss: 3.7357 | LR: 4.15e-04 | 1327.3ms/step | 48.2 samples/s
394
+ Step 2790/10001 | Loss: 3.7692 | LR: 4.14e-04 | 1328.1ms/step | 48.2 samples/s
395
+ Step 2800/10001 | Loss: 3.7441 | LR: 4.14e-04 | 1329.4ms/step | 48.1 samples/s
396
+ Step 2810/10001 | Loss: 3.7217 | LR: 4.13e-04 | 1330.0ms/step | 48.1 samples/s
397
+ Step 2820/10001 | Loss: 3.7459 | LR: 4.13e-04 | 1329.7ms/step | 48.1 samples/s
398
+ Step 2830/10001 | Loss: 3.7500 | LR: 4.12e-04 | 1331.2ms/step | 48.1 samples/s
399
+ Step 2840/10001 | Loss: 3.7425 | LR: 4.11e-04 | 1329.7ms/step | 48.1 samples/s
400
+ Step 2850/10001 | Loss: 3.7343 | LR: 4.11e-04 | 1327.2ms/step | 48.2 samples/s
401
+ Step 2860/10001 | Loss: 3.7357 | LR: 4.10e-04 | 1327.8ms/step | 48.2 samples/s
402
+ Step 2870/10001 | Loss: 3.7538 | LR: 4.09e-04 | 1329.1ms/step | 48.2 samples/s
403
+ Step 2880/10001 | Loss: 3.7572 | LR: 4.09e-04 | 1329.6ms/step | 48.1 samples/s
404
+ Step 2890/10001 | Loss: 3.7578 | LR: 4.08e-04 | 1328.6ms/step | 48.2 samples/s
405
+ Step 2900/10001 | Loss: 3.7148 | LR: 4.08e-04 | 1327.2ms/step | 48.2 samples/s
406
+ Step 2910/10001 | Loss: 3.7160 | LR: 4.07e-04 | 1326.4ms/step | 48.3 samples/s
407
+ Step 2920/10001 | Loss: 3.7274 | LR: 4.06e-04 | 1328.1ms/step | 48.2 samples/s
408
+ Step 2930/10001 | Loss: 3.7387 | LR: 4.06e-04 | 1328.5ms/step | 48.2 samples/s
409
+ Step 2940/10001 | Loss: 3.6942 | LR: 4.05e-04 | 1325.7ms/step | 48.3 samples/s
410
+ Step 2950/10001 | Loss: 3.7003 | LR: 4.05e-04 | 1327.7ms/step | 48.2 samples/s
411
+ Step 2960/10001 | Loss: 3.7121 | LR: 4.04e-04 | 1328.1ms/step | 48.2 samples/s
412
+ Step 2970/10001 | Loss: 3.7170 | LR: 4.03e-04 | 1328.5ms/step | 48.2 samples/s
413
+ Step 2980/10001 | Loss: 3.7173 | LR: 4.03e-04 | 1329.6ms/step | 48.1 samples/s
414
+ Step 2990/10001 | Loss: 3.7278 | LR: 4.02e-04 | 1329.6ms/step | 48.1 samples/s
415
+ Step 3000/10001 | Loss: 3.6999 | LR: 4.01e-04 | 1329.6ms/step | 48.1 samples/s
416
+ Validation at step 3000: Loss = 3.7117, Perplexity = 40.92
417
+ Step 3010/10001 | Loss: 3.7453 | LR: 4.01e-04 | 26731.2ms/step | 2.4 samples/s
418
+ Step 3020/10001 | Loss: 3.7209 | LR: 4.00e-04 | 1329.3ms/step | 48.1 samples/s
419
+ Step 3030/10001 | Loss: 3.7065 | LR: 4.00e-04 | 1329.1ms/step | 48.2 samples/s
420
+ Step 3040/10001 | Loss: 3.6880 | LR: 3.99e-04 | 1331.1ms/step | 48.1 samples/s
421
+ Step 3050/10001 | Loss: 3.7042 | LR: 3.98e-04 | 1331.1ms/step | 48.1 samples/s
422
+ Step 3060/10001 | Loss: 3.7146 | LR: 3.98e-04 | 1329.3ms/step | 48.1 samples/s
423
+ Step 3070/10001 | Loss: 3.6723 | LR: 3.97e-04 | 1326.7ms/step | 48.2 samples/s
424
+ Step 3080/10001 | Loss: 3.6804 | LR: 3.96e-04 | 1328.7ms/step | 48.2 samples/s
425
+ Step 3090/10001 | Loss: 3.7053 | LR: 3.96e-04 | 1328.9ms/step | 48.2 samples/s
426
+ Step 3100/10001 | Loss: 3.6787 | LR: 3.95e-04 | 1331.1ms/step | 48.1 samples/s
427
+ Step 3110/10001 | Loss: 3.7284 | LR: 3.94e-04 | 1329.3ms/step | 48.1 samples/s
428
+ Step 3120/10001 | Loss: 3.7093 | LR: 3.94e-04 | 1330.8ms/step | 48.1 samples/s
429
+ Step 3130/10001 | Loss: 3.6857 | LR: 3.93e-04 | 1329.5ms/step | 48.1 samples/s
430
+ Step 3140/10001 | Loss: 3.7012 | LR: 3.92e-04 | 1330.6ms/step | 48.1 samples/s
431
+ Step 3150/10001 | Loss: 3.6951 | LR: 3.92e-04 | 1329.4ms/step | 48.1 samples/s
432
+ Step 3160/10001 | Loss: 3.6646 | LR: 3.91e-04 | 1331.5ms/step | 48.1 samples/s
433
+ Step 3170/10001 | Loss: 3.6664 | LR: 3.90e-04 | 1328.0ms/step | 48.2 samples/s
434
+ Step 3180/10001 | Loss: 3.6888 | LR: 3.90e-04 | 1328.9ms/step | 48.2 samples/s
435
+ Step 3190/10001 | Loss: 3.7068 | LR: 3.89e-04 | 1329.9ms/step | 48.1 samples/s
436
+ Step 3200/10001 | Loss: 3.6771 | LR: 3.89e-04 | 1331.5ms/step | 48.1 samples/s
437
+ Step 3210/10001 | Loss: 3.6701 | LR: 3.88e-04 | 1328.7ms/step | 48.2 samples/s
438
+ Step 3220/10001 | Loss: 3.6912 | LR: 3.87e-04 | 1329.4ms/step | 48.1 samples/s
439
+ Step 3230/10001 | Loss: 3.6751 | LR: 3.87e-04 | 1329.3ms/step | 48.1 samples/s
440
+ Step 3240/10001 | Loss: 3.6653 | LR: 3.86e-04 | 1329.0ms/step | 48.2 samples/s
441
+ Step 3250/10001 | Loss: 3.6689 | LR: 3.85e-04 | 1329.0ms/step | 48.2 samples/s
442
+ Validation at step 3250: Loss = 3.6796, Perplexity = 39.63
443
+ Step 3260/10001 | Loss: 3.6887 | LR: 3.85e-04 | 26708.1ms/step | 2.4 samples/s
444
+ Step 3270/10001 | Loss: 3.6747 | LR: 3.84e-04 | 1331.4ms/step | 48.1 samples/s
445
+ Step 3280/10001 | Loss: 3.6798 | LR: 3.83e-04 | 1414.6ms/step | 45.2 samples/s
446
+ Step 3290/10001 | Loss: 3.6612 | LR: 3.83e-04 | 1328.5ms/step | 48.2 samples/s
447
+ Step 3300/10001 | Loss: 3.6864 | LR: 3.82e-04 | 1329.0ms/step | 48.2 samples/s
448
+ Step 3310/10001 | Loss: 3.6736 | LR: 3.81e-04 | 1327.2ms/step | 48.2 samples/s
449
+ Step 3320/10001 | Loss: 3.6604 | LR: 3.80e-04 | 1328.7ms/step | 48.2 samples/s
450
+ Step 3330/10001 | Loss: 3.6333 | LR: 3.80e-04 | 1328.8ms/step | 48.2 samples/s
451
+ Step 3340/10001 | Loss: 3.6478 | LR: 3.79e-04 | 1330.3ms/step | 48.1 samples/s
452
+ Step 3350/10001 | Loss: 3.6453 | LR: 3.78e-04 | 1331.3ms/step | 48.1 samples/s
453
+ Step 3360/10001 | Loss: 3.6542 | LR: 3.78e-04 | 1329.3ms/step | 48.1 samples/s
454
+ Step 3370/10001 | Loss: 3.6636 | LR: 3.77e-04 | 1327.4ms/step | 48.2 samples/s
455
+ Step 3380/10001 | Loss: 3.6423 | LR: 3.76e-04 | 1328.7ms/step | 48.2 samples/s
456
+ Step 3390/10001 | Loss: 3.6530 | LR: 3.76e-04 | 1329.7ms/step | 48.1 samples/s
457
+ Step 3400/10001 | Loss: 3.6318 | LR: 3.75e-04 | 1327.3ms/step | 48.2 samples/s
458
+ Step 3410/10001 | Loss: 3.6574 | LR: 3.74e-04 | 1328.8ms/step | 48.2 samples/s
459
+ Step 3420/10001 | Loss: 3.6492 | LR: 3.74e-04 | 1329.1ms/step | 48.2 samples/s
460
+ Step 3430/10001 | Loss: 3.6328 | LR: 3.73e-04 | 1329.3ms/step | 48.1 samples/s
461
+ Step 3440/10001 | Loss: 3.6391 | LR: 3.72e-04 | 1327.7ms/step | 48.2 samples/s
462
+ Step 3450/10001 | Loss: 3.6603 | LR: 3.72e-04 | 1327.4ms/step | 48.2 samples/s
463
+ Step 3460/10001 | Loss: 3.6581 | LR: 3.71e-04 | 1327.4ms/step | 48.2 samples/s
464
+ Step 3470/10001 | Loss: 3.6316 | LR: 3.70e-04 | 1327.9ms/step | 48.2 samples/s
465
+ Step 3480/10001 | Loss: 3.6406 | LR: 3.69e-04 | 1328.2ms/step | 48.2 samples/s
466
+ Step 3490/10001 | Loss: 3.6368 | LR: 3.69e-04 | 1328.6ms/step | 48.2 samples/s
467
+ Step 3500/10001 | Loss: 3.6179 | LR: 3.68e-04 | 1327.7ms/step | 48.2 samples/s
468
+ Validation at step 3500: Loss = 3.6487, Perplexity = 38.43
469
+ Step 3510/10001 | Loss: 3.6160 | LR: 3.67e-04 | 26709.1ms/step | 2.4 samples/s
470
+ Step 3520/10001 | Loss: 3.6119 | LR: 3.67e-04 | 1328.2ms/step | 48.2 samples/s
471
+ Step 3530/10001 | Loss: 3.6709 | LR: 3.66e-04 | 1333.6ms/step | 48.0 samples/s
472
+ Step 3540/10001 | Loss: 3.6466 | LR: 3.65e-04 | 1329.6ms/step | 48.1 samples/s
473
+ Step 3550/10001 | Loss: 3.6554 | LR: 3.65e-04 | 1330.7ms/step | 48.1 samples/s
474
+ Step 3560/10001 | Loss: 3.6450 | LR: 3.64e-04 | 1329.1ms/step | 48.2 samples/s
475
+ Step 3570/10001 | Loss: 3.6323 | LR: 3.63e-04 | 1327.9ms/step | 48.2 samples/s
476
+ Step 3580/10001 | Loss: 3.6491 | LR: 3.62e-04 | 1326.3ms/step | 48.3 samples/s
477
+ Step 3590/10001 | Loss: 3.6254 | LR: 3.62e-04 | 1330.1ms/step | 48.1 samples/s
478
+ Step 3600/10001 | Loss: 3.6523 | LR: 3.61e-04 | 1329.1ms/step | 48.2 samples/s
479
+ Step 3610/10001 | Loss: 3.6212 | LR: 3.60e-04 | 1330.1ms/step | 48.1 samples/s
480
+ Step 3620/10001 | Loss: 3.6433 | LR: 3.60e-04 | 1330.8ms/step | 48.1 samples/s
481
+ Step 3630/10001 | Loss: 3.6147 | LR: 3.59e-04 | 1331.1ms/step | 48.1 samples/s
482
+ Step 3640/10001 | Loss: 3.6461 | LR: 3.58e-04 | 1330.7ms/step | 48.1 samples/s
483
+ Step 3650/10001 | Loss: 3.6488 | LR: 3.57e-04 | 1330.7ms/step | 48.1 samples/s
484
+ Step 3660/10001 | Loss: 3.6272 | LR: 3.57e-04 | 1329.0ms/step | 48.2 samples/s
485
+ Step 3670/10001 | Loss: 3.6165 | LR: 3.56e-04 | 1329.8ms/step | 48.1 samples/s
486
+ Step 3680/10001 | Loss: 3.6055 | LR: 3.55e-04 | 1327.9ms/step | 48.2 samples/s
487
+ Step 3690/10001 | Loss: 3.6485 | LR: 3.55e-04 | 1328.3ms/step | 48.2 samples/s
488
+ Step 3700/10001 | Loss: 3.6271 | LR: 3.54e-04 | 1327.6ms/step | 48.2 samples/s
489
+ Step 3710/10001 | Loss: 3.6331 | LR: 3.53e-04 | 1328.6ms/step | 48.2 samples/s
490
+ Step 3720/10001 | Loss: 3.5996 | LR: 3.52e-04 | 1329.0ms/step | 48.2 samples/s
491
+ Step 3730/10001 | Loss: 3.6224 | LR: 3.52e-04 | 1328.7ms/step | 48.2 samples/s
492
+ Step 3740/10001 | Loss: 3.5997 | LR: 3.51e-04 | 1328.7ms/step | 48.2 samples/s
493
+ Step 3750/10001 | Loss: 3.6219 | LR: 3.50e-04 | 1328.7ms/step | 48.2 samples/s
494
+ Validation at step 3750: Loss = 3.6213, Perplexity = 37.39
495
+ Step 3760/10001 | Loss: 3.6650 | LR: 3.50e-04 | 26717.0ms/step | 2.4 samples/s
496
+ Step 3770/10001 | Loss: 3.6357 | LR: 3.49e-04 | 1330.9ms/step | 48.1 samples/s
497
+ Step 3780/10001 | Loss: 3.6536 | LR: 3.48e-04 | 1330.3ms/step | 48.1 samples/s
498
+ Step 3790/10001 | Loss: 3.5935 | LR: 3.47e-04 | 1327.2ms/step | 48.2 samples/s
499
+ Step 3800/10001 | Loss: 3.6073 | LR: 3.47e-04 | 1330.2ms/step | 48.1 samples/s
500
+ Step 3810/10001 | Loss: 3.6065 | LR: 3.46e-04 | 1329.9ms/step | 48.1 samples/s
501
+ Step 3820/10001 | Loss: 3.5799 | LR: 3.45e-04 | 1331.0ms/step | 48.1 samples/s
502
+ Step 3830/10001 | Loss: 3.6168 | LR: 3.44e-04 | 1328.2ms/step | 48.2 samples/s
503
+ Step 3840/10001 | Loss: 3.6337 | LR: 3.44e-04 | 1332.3ms/step | 48.0 samples/s
504
+ Step 3850/10001 | Loss: 3.6040 | LR: 3.43e-04 | 1330.7ms/step | 48.1 samples/s
505
+ Step 3860/10001 | Loss: 3.6117 | LR: 3.42e-04 | 1327.8ms/step | 48.2 samples/s
506
+ Step 3870/10001 | Loss: 3.5972 | LR: 3.41e-04 | 1328.6ms/step | 48.2 samples/s
507
+ Step 3880/10001 | Loss: 3.6371 | LR: 3.41e-04 | 1327.8ms/step | 48.2 samples/s
508
+ Step 3890/10001 | Loss: 3.6110 | LR: 3.40e-04 | 1327.9ms/step | 48.2 samples/s
509
+ Step 3900/10001 | Loss: 3.5988 | LR: 3.39e-04 | 1327.9ms/step | 48.2 samples/s
510
+ Step 3910/10001 | Loss: 3.6245 | LR: 3.39e-04 | 1328.9ms/step | 48.2 samples/s
511
+ Step 3920/10001 | Loss: 3.5937 | LR: 3.38e-04 | 1329.0ms/step | 48.2 samples/s
512
+ Step 3930/10001 | Loss: 3.5685 | LR: 3.37e-04 | 1329.8ms/step | 48.1 samples/s
513
+ Step 3940/10001 | Loss: 3.5947 | LR: 3.36e-04 | 1327.6ms/step | 48.2 samples/s
514
+ Step 3950/10001 | Loss: 3.6013 | LR: 3.36e-04 | 1329.0ms/step | 48.2 samples/s
515
+ Step 3960/10001 | Loss: 3.6086 | LR: 3.35e-04 | 1329.9ms/step | 48.1 samples/s
516
+ Step 3970/10001 | Loss: 3.5980 | LR: 3.34e-04 | 1328.6ms/step | 48.2 samples/s
517
+ Step 3980/10001 | Loss: 3.6228 | LR: 3.33e-04 | 1326.7ms/step | 48.2 samples/s
518
+ Step 3990/10001 | Loss: 3.5895 | LR: 3.33e-04 | 1326.9ms/step | 48.2 samples/s
519
+ Step 4000/10001 | Loss: 3.5702 | LR: 3.32e-04 | 1324.2ms/step | 48.3 samples/s
520
+ Validation at step 4000: Loss = 3.5950, Perplexity = 36.42
521
+ Step 4010/10001 | Loss: 3.5844 | LR: 3.31e-04 | 26808.2ms/step | 2.4 samples/s
522
+ Step 4020/10001 | Loss: 3.5899 | LR: 3.30e-04 | 1329.6ms/step | 48.1 samples/s
523
+ Step 4030/10001 | Loss: 3.5587 | LR: 3.30e-04 | 1330.4ms/step | 48.1 samples/s
524
+ Step 4040/10001 | Loss: 3.6066 | LR: 3.29e-04 | 1332.4ms/step | 48.0 samples/s
525
+ Step 4050/10001 | Loss: 3.5911 | LR: 3.28e-04 | 1333.2ms/step | 48.0 samples/s
526
+ Step 4060/10001 | Loss: 3.5810 | LR: 3.27e-04 | 1332.7ms/step | 48.0 samples/s
527
+ Step 4070/10001 | Loss: 3.5813 | LR: 3.27e-04 | 1328.2ms/step | 48.2 samples/s
528
+ Step 4080/10001 | Loss: 3.5981 | LR: 3.26e-04 | 1326.3ms/step | 48.3 samples/s
529
+ Step 4090/10001 | Loss: 3.5750 | LR: 3.25e-04 | 1326.0ms/step | 48.3 samples/s
530
+ Step 4100/10001 | Loss: 3.5760 | LR: 3.24e-04 | 1326.8ms/step | 48.2 samples/s
531
+ Step 4110/10001 | Loss: 3.5807 | LR: 3.24e-04 | 1327.0ms/step | 48.2 samples/s
532
+ Step 4120/10001 | Loss: 3.6144 | LR: 3.23e-04 | 1325.7ms/step | 48.3 samples/s
533
+ Step 4130/10001 | Loss: 3.5833 | LR: 3.22e-04 | 1325.3ms/step | 48.3 samples/s
534
+ Step 4140/10001 | Loss: 3.5676 | LR: 3.21e-04 | 1325.4ms/step | 48.3 samples/s
535
+ Step 4150/10001 | Loss: 3.5763 | LR: 3.20e-04 | 1325.8ms/step | 48.3 samples/s
536
+ Step 4160/10001 | Loss: 3.5642 | LR: 3.20e-04 | 1329.5ms/step | 48.1 samples/s
537
+ Step 4170/10001 | Loss: 3.5914 | LR: 3.19e-04 | 1332.5ms/step | 48.0 samples/s
538
+ Step 4180/10001 | Loss: 3.6044 | LR: 3.18e-04 | 1329.1ms/step | 48.2 samples/s
539
+ Step 4190/10001 | Loss: 3.5603 | LR: 3.17e-04 | 1327.9ms/step | 48.2 samples/s
540
+ Step 4200/10001 | Loss: 3.5975 | LR: 3.17e-04 | 1330.5ms/step | 48.1 samples/s
541
+ Step 4210/10001 | Loss: 3.5872 | LR: 3.16e-04 | 1331.1ms/step | 48.1 samples/s
542
+ Step 4220/10001 | Loss: 3.5772 | LR: 3.15e-04 | 1329.4ms/step | 48.1 samples/s
543
+ Step 4230/10001 | Loss: 3.5805 | LR: 3.14e-04 | 1329.1ms/step | 48.2 samples/s
544
+ Step 4240/10001 | Loss: 3.5846 | LR: 3.14e-04 | 1329.8ms/step | 48.1 samples/s
545
+ Step 4250/10001 | Loss: 3.5493 | LR: 3.13e-04 | 1330.0ms/step | 48.1 samples/s
546
+ Validation at step 4250: Loss = 3.5721, Perplexity = 35.59
547
+ Step 4260/10001 | Loss: 3.5551 | LR: 3.12e-04 | 26697.8ms/step | 2.4 samples/s
548
+ Step 4270/10001 | Loss: 3.5749 | LR: 3.11e-04 | 1327.2ms/step | 48.2 samples/s
549
+ Step 4280/10001 | Loss: 3.5381 | LR: 3.11e-04 | 1327.4ms/step | 48.2 samples/s
550
+ Step 4290/10001 | Loss: 3.5565 | LR: 3.10e-04 | 1327.9ms/step | 48.2 samples/s
551
+ Step 4300/10001 | Loss: 3.5413 | LR: 3.09e-04 | 1328.8ms/step | 48.2 samples/s
552
+ Step 4310/10001 | Loss: 3.5668 | LR: 3.08e-04 | 1327.6ms/step | 48.2 samples/s
553
+ Step 4320/10001 | Loss: 3.5868 | LR: 3.07e-04 | 1328.0ms/step | 48.2 samples/s
554
+ Step 4330/10001 | Loss: 3.5715 | LR: 3.07e-04 | 1327.6ms/step | 48.2 samples/s
555
+ Step 4340/10001 | Loss: 3.5613 | LR: 3.06e-04 | 1329.0ms/step | 48.2 samples/s
556
+ Step 4350/10001 | Loss: 3.5622 | LR: 3.05e-04 | 1329.1ms/step | 48.2 samples/s
557
+ Step 4360/10001 | Loss: 3.5371 | LR: 3.04e-04 | 1329.0ms/step | 48.2 samples/s
558
+ Step 4370/10001 | Loss: 3.5542 | LR: 3.04e-04 | 1329.1ms/step | 48.2 samples/s
559
+ Step 4380/10001 | Loss: 3.5748 | LR: 3.03e-04 | 1328.1ms/step | 48.2 samples/s
560
+ Step 4390/10001 | Loss: 3.5804 | LR: 3.02e-04 | 1326.7ms/step | 48.2 samples/s
561
+ Step 4400/10001 | Loss: 3.5536 | LR: 3.01e-04 | 1327.7ms/step | 48.2 samples/s
562
+ Step 4410/10001 | Loss: 3.5580 | LR: 3.00e-04 | 1325.3ms/step | 48.3 samples/s
563
+ Step 4420/10001 | Loss: 3.5221 | LR: 3.00e-04 | 1325.8ms/step | 48.3 samples/s
564
+ Step 4430/10001 | Loss: 3.5812 | LR: 2.99e-04 | 1328.4ms/step | 48.2 samples/s
565
+ Step 4440/10001 | Loss: 3.5270 | LR: 2.98e-04 | 1329.1ms/step | 48.2 samples/s
566
+ Step 4450/10001 | Loss: 3.5619 | LR: 2.97e-04 | 1327.2ms/step | 48.2 samples/s
567
+ Step 4460/10001 | Loss: 3.5626 | LR: 2.97e-04 | 1327.3ms/step | 48.2 samples/s
568
+ Step 4470/10001 | Loss: 3.5437 | LR: 2.96e-04 | 1330.3ms/step | 48.1 samples/s
569
+ Step 4480/10001 | Loss: 3.5333 | LR: 2.95e-04 | 1331.5ms/step | 48.1 samples/s
570
+ Step 4490/10001 | Loss: 3.5839 | LR: 2.94e-04 | 1332.1ms/step | 48.0 samples/s
571
+ Step 4500/10001 | Loss: 3.5379 | LR: 2.93e-04 | 1329.6ms/step | 48.1 samples/s
572
+ Validation at step 4500: Loss = 3.5500, Perplexity = 34.81
573
+ Step 4510/10001 | Loss: 3.5377 | LR: 2.93e-04 | 26723.2ms/step | 2.4 samples/s
574
+ Step 4520/10001 | Loss: 3.5527 | LR: 2.92e-04 | 1333.0ms/step | 48.0 samples/s
575
+ Step 4530/10001 | Loss: 3.5254 | LR: 2.91e-04 | 1331.9ms/step | 48.1 samples/s
576
+ Step 4540/10001 | Loss: 3.5511 | LR: 2.90e-04 | 1329.1ms/step | 48.2 samples/s
577
+ Step 4550/10001 | Loss: 3.5546 | LR: 2.90e-04 | 1330.0ms/step | 48.1 samples/s
578
+ Step 4560/10001 | Loss: 3.5505 | LR: 2.89e-04 | 1328.9ms/step | 48.2 samples/s
579
+ Step 4570/10001 | Loss: 3.5413 | LR: 2.88e-04 | 1328.6ms/step | 48.2 samples/s
580
+ Step 4580/10001 | Loss: 3.5490 | LR: 2.87e-04 | 1328.6ms/step | 48.2 samples/s
581
+ Step 4590/10001 | Loss: 3.5344 | LR: 2.86e-04 | 1326.6ms/step | 48.2 samples/s
582
+ Step 4600/10001 | Loss: 3.5251 | LR: 2.86e-04 | 1327.1ms/step | 48.2 samples/s
583
+ Step 4610/10001 | Loss: 3.5123 | LR: 2.85e-04 | 1328.1ms/step | 48.2 samples/s
584
+ Step 4620/10001 | Loss: 3.5456 | LR: 2.84e-04 | 1327.5ms/step | 48.2 samples/s
585
+ Step 4630/10001 | Loss: 3.5366 | LR: 2.83e-04 | 1326.4ms/step | 48.2 samples/s
586
+ Step 4640/10001 | Loss: 3.5539 | LR: 2.82e-04 | 1328.1ms/step | 48.2 samples/s
587
+ Step 4650/10001 | Loss: 3.5448 | LR: 2.82e-04 | 1328.0ms/step | 48.2 samples/s
588
+ Step 4660/10001 | Loss: 3.5418 | LR: 2.81e-04 | 1329.8ms/step | 48.1 samples/s
589
+ Step 4670/10001 | Loss: 3.5091 | LR: 2.80e-04 | 1330.9ms/step | 48.1 samples/s
590
+ Step 4680/10001 | Loss: 3.5163 | LR: 2.79e-04 | 1331.4ms/step | 48.1 samples/s
591
+ Step 4690/10001 | Loss: 3.5077 | LR: 2.79e-04 | 1333.0ms/step | 48.0 samples/s
592
+ Step 4700/10001 | Loss: 3.4964 | LR: 2.78e-04 | 1331.0ms/step | 48.1 samples/s
593
+ Step 4710/10001 | Loss: 3.5289 | LR: 2.77e-04 | 1328.1ms/step | 48.2 samples/s
594
+ Step 4720/10001 | Loss: 3.5751 | LR: 2.76e-04 | 1328.2ms/step | 48.2 samples/s
595
+ Step 4730/10001 | Loss: 3.5336 | LR: 2.75e-04 | 1331.7ms/step | 48.1 samples/s
596
+ Step 4740/10001 | Loss: 3.5206 | LR: 2.75e-04 | 1414.6ms/step | 45.2 samples/s
597
+ Step 4750/10001 | Loss: 3.5185 | LR: 2.74e-04 | 1329.6ms/step | 48.1 samples/s
598
+ Validation at step 4750: Loss = 3.5308, Perplexity = 34.15
599
+ Step 4760/10001 | Loss: 3.5483 | LR: 2.73e-04 | 26706.0ms/step | 2.4 samples/s
600
+ Step 4770/10001 | Loss: 3.5384 | LR: 2.72e-04 | 1328.8ms/step | 48.2 samples/s
601
+ Step 4780/10001 | Loss: 3.5446 | LR: 2.71e-04 | 1327.3ms/step | 48.2 samples/s
602
+ Step 4790/10001 | Loss: 3.5275 | LR: 2.71e-04 | 1324.9ms/step | 48.3 samples/s
603
+ Step 4800/10001 | Loss: 3.5189 | LR: 2.70e-04 | 1326.0ms/step | 48.3 samples/s
604
+ Step 4810/10001 | Loss: 3.5223 | LR: 2.69e-04 | 1324.9ms/step | 48.3 samples/s
605
+ Step 4820/10001 | Loss: 3.5179 | LR: 2.68e-04 | 1324.2ms/step | 48.3 samples/s
606
+ Step 4830/10001 | Loss: 3.5015 | LR: 2.67e-04 | 1325.7ms/step | 48.3 samples/s
607
+ Step 4840/10001 | Loss: 3.5556 | LR: 2.67e-04 | 1327.1ms/step | 48.2 samples/s
608
+ Step 4850/10001 | Loss: 3.5281 | LR: 2.66e-04 | 1328.7ms/step | 48.2 samples/s
609
+ Step 4860/10001 | Loss: 3.5322 | LR: 2.65e-04 | 1328.3ms/step | 48.2 samples/s
610
+ Step 4870/10001 | Loss: 3.5454 | LR: 2.64e-04 | 1328.0ms/step | 48.2 samples/s
611
+ Step 4880/10001 | Loss: 3.5032 | LR: 2.64e-04 | 1329.0ms/step | 48.2 samples/s
612
+ Step 4890/10001 | Loss: 3.5259 | LR: 2.63e-04 | 1329.0ms/step | 48.2 samples/s
613
+ Step 4900/10001 | Loss: 3.5191 | LR: 2.62e-04 | 1328.7ms/step | 48.2 samples/s
614
+ Step 4910/10001 | Loss: 3.5222 | LR: 2.61e-04 | 1326.9ms/step | 48.2 samples/s
615
+ Step 4920/10001 | Loss: 3.4939 | LR: 2.60e-04 | 1327.4ms/step | 48.2 samples/s
616
+ Step 4930/10001 | Loss: 3.4869 | LR: 2.60e-04 | 1331.4ms/step | 48.1 samples/s
617
+ Step 4940/10001 | Loss: 3.5133 | LR: 2.59e-04 | 1330.5ms/step | 48.1 samples/s
618
+ Step 4950/10001 | Loss: 3.5511 | LR: 2.58e-04 | 1327.4ms/step | 48.2 samples/s
619
+ Step 4960/10001 | Loss: 3.5286 | LR: 2.57e-04 | 1325.8ms/step | 48.3 samples/s
620
+ Step 4970/10001 | Loss: 3.5014 | LR: 2.56e-04 | 1325.3ms/step | 48.3 samples/s
621
+ Step 4980/10001 | Loss: 3.5162 | LR: 2.56e-04 | 1325.1ms/step | 48.3 samples/s
622
+ Step 4990/10001 | Loss: 3.5201 | LR: 2.55e-04 | 1327.4ms/step | 48.2 samples/s
623
+ Step 5000/10001 | Loss: 3.5144 | LR: 2.54e-04 | 1327.6ms/step | 48.2 samples/s
624
+ Validation at step 5000: Loss = 3.5115, Perplexity = 33.50
625
+ Saved checkpoint to checkpoints/run_20260430_021312/checkpoint-5000
626
+ Step 5010/10001 | Loss: 3.5418 | LR: 2.53e-04 | 27252.0ms/step | 2.3 samples/s
627
+ Step 5020/10001 | Loss: 3.4961 | LR: 2.52e-04 | 1329.0ms/step | 48.2 samples/s
628
+ Step 5030/10001 | Loss: 3.4768 | LR: 2.52e-04 | 1328.0ms/step | 48.2 samples/s
629
+ Step 5040/10001 | Loss: 3.5281 | LR: 2.51e-04 | 1329.2ms/step | 48.1 samples/s
630
+ Step 5050/10001 | Loss: 3.5214 | LR: 2.50e-04 | 1330.4ms/step | 48.1 samples/s
631
+ Step 5060/10001 | Loss: 3.5115 | LR: 2.49e-04 | 1330.1ms/step | 48.1 samples/s
632
+ Step 5070/10001 | Loss: 3.5245 | LR: 2.48e-04 | 1331.2ms/step | 48.1 samples/s
633
+ Step 5080/10001 | Loss: 3.5003 | LR: 2.48e-04 | 1333.9ms/step | 48.0 samples/s
634
+ Step 5090/10001 | Loss: 3.5171 | LR: 2.47e-04 | 1333.4ms/step | 48.0 samples/s
635
+ Step 5100/10001 | Loss: 3.4871 | LR: 2.46e-04 | 1333.2ms/step | 48.0 samples/s
636
+ Step 5110/10001 | Loss: 3.5054 | LR: 2.45e-04 | 1333.5ms/step | 48.0 samples/s
637
+ Step 5120/10001 | Loss: 3.5037 | LR: 2.44e-04 | 1334.6ms/step | 48.0 samples/s
638
+ Step 5130/10001 | Loss: 3.5051 | LR: 2.44e-04 | 1336.9ms/step | 47.9 samples/s
639
+ Step 5140/10001 | Loss: 3.5266 | LR: 2.43e-04 | 1334.0ms/step | 48.0 samples/s
640
+ Step 5150/10001 | Loss: 3.4952 | LR: 2.42e-04 | 1332.4ms/step | 48.0 samples/s
641
+ Step 5160/10001 | Loss: 3.5064 | LR: 2.41e-04 | 1329.3ms/step | 48.1 samples/s
642
+ Step 5170/10001 | Loss: 3.5178 | LR: 2.41e-04 | 1329.8ms/step | 48.1 samples/s
643
+ Step 5180/10001 | Loss: 3.5149 | LR: 2.40e-04 | 1331.5ms/step | 48.1 samples/s
644
+ Step 5190/10001 | Loss: 3.4944 | LR: 2.39e-04 | 1332.3ms/step | 48.0 samples/s
645
+ Step 5200/10001 | Loss: 3.4944 | LR: 2.38e-04 | 1331.4ms/step | 48.1 samples/s
646
+ Step 5210/10001 | Loss: 3.5087 | LR: 2.37e-04 | 1333.7ms/step | 48.0 samples/s
647
+ Step 5220/10001 | Loss: 3.4971 | LR: 2.37e-04 | 1333.0ms/step | 48.0 samples/s
648
+ Step 5230/10001 | Loss: 3.5080 | LR: 2.36e-04 | 1333.5ms/step | 48.0 samples/s
649
+ Step 5240/10001 | Loss: 3.5040 | LR: 2.35e-04 | 1332.3ms/step | 48.0 samples/s
650
+ Step 5250/10001 | Loss: 3.4699 | LR: 2.34e-04 | 1330.8ms/step | 48.1 samples/s
651
+ Validation at step 5250: Loss = 3.4944, Perplexity = 32.93
652
+ Step 5260/10001 | Loss: 3.4835 | LR: 2.33e-04 | 26810.4ms/step | 2.4 samples/s
653
+ Step 5270/10001 | Loss: 3.4917 | LR: 2.33e-04 | 1335.6ms/step | 47.9 samples/s
654
+ Step 5280/10001 | Loss: 3.4787 | LR: 2.32e-04 | 1331.8ms/step | 48.1 samples/s
655
+ Step 5290/10001 | Loss: 3.4763 | LR: 2.31e-04 | 1333.5ms/step | 48.0 samples/s
656
+ Step 5300/10001 | Loss: 3.5288 | LR: 2.30e-04 | 1331.0ms/step | 48.1 samples/s
657
+ Step 5310/10001 | Loss: 3.4840 | LR: 2.29e-04 | 1331.4ms/step | 48.1 samples/s
658
+ Step 5320/10001 | Loss: 3.4703 | LR: 2.29e-04 | 1330.0ms/step | 48.1 samples/s
659
+ Step 5330/10001 | Loss: 3.4709 | LR: 2.28e-04 | 1331.0ms/step | 48.1 samples/s
660
+ Step 5340/10001 | Loss: 3.4733 | LR: 2.27e-04 | 1331.0ms/step | 48.1 samples/s
661
+ Step 5350/10001 | Loss: 3.5078 | LR: 2.26e-04 | 1330.5ms/step | 48.1 samples/s
662
+ Step 5360/10001 | Loss: 3.4850 | LR: 2.25e-04 | 1327.7ms/step | 48.2 samples/s
663
+ Step 5370/10001 | Loss: 3.4905 | LR: 2.25e-04 | 1326.2ms/step | 48.3 samples/s
664
+ Step 5380/10001 | Loss: 3.4753 | LR: 2.24e-04 | 1329.4ms/step | 48.1 samples/s
665
+ Step 5390/10001 | Loss: 3.4726 | LR: 2.23e-04 | 1329.1ms/step | 48.2 samples/s
666
+ Step 5400/10001 | Loss: 3.5215 | LR: 2.22e-04 | 1328.2ms/step | 48.2 samples/s
667
+ Step 5410/10001 | Loss: 3.4738 | LR: 2.22e-04 | 1330.0ms/step | 48.1 samples/s
668
+ Step 5420/10001 | Loss: 3.4907 | LR: 2.21e-04 | 1330.9ms/step | 48.1 samples/s
669
+ Step 5430/10001 | Loss: 3.4963 | LR: 2.20e-04 | 1330.9ms/step | 48.1 samples/s
670
+ Step 5440/10001 | Loss: 3.5013 | LR: 2.19e-04 | 1331.5ms/step | 48.1 samples/s
671
+ Step 5450/10001 | Loss: 3.4821 | LR: 2.18e-04 | 1333.9ms/step | 48.0 samples/s
672
+ Step 5460/10001 | Loss: 3.5005 | LR: 2.18e-04 | 1334.2ms/step | 48.0 samples/s
673
+ Step 5470/10001 | Loss: 3.5146 | LR: 2.17e-04 | 1415.4ms/step | 45.2 samples/s
674
+ Step 5480/10001 | Loss: 3.4957 | LR: 2.16e-04 | 1331.3ms/step | 48.1 samples/s
675
+ Step 5490/10001 | Loss: 3.4696 | LR: 2.15e-04 | 1332.7ms/step | 48.0 samples/s
676
+ Step 5500/10001 | Loss: 3.4716 | LR: 2.14e-04 | 1331.4ms/step | 48.1 samples/s
677
+ Validation at step 5500: Loss = 3.4791, Perplexity = 32.43
678
+ Step 5510/10001 | Loss: 3.4789 | LR: 2.14e-04 | 26793.3ms/step | 2.4 samples/s
679
+ Step 5520/10001 | Loss: 3.4918 | LR: 2.13e-04 | 1331.5ms/step | 48.1 samples/s
680
+ Step 5530/10001 | Loss: 3.4970 | LR: 2.12e-04 | 1330.6ms/step | 48.1 samples/s
681
+ Step 5540/10001 | Loss: 3.4945 | LR: 2.11e-04 | 1330.9ms/step | 48.1 samples/s
682
+ Step 5550/10001 | Loss: 3.4746 | LR: 2.11e-04 | 1333.2ms/step | 48.0 samples/s
683
+ Step 5560/10001 | Loss: 3.4794 | LR: 2.10e-04 | 1332.6ms/step | 48.0 samples/s
684
+ Step 5570/10001 | Loss: 3.4511 | LR: 2.09e-04 | 1333.5ms/step | 48.0 samples/s
685
+ Step 5580/10001 | Loss: 3.4827 | LR: 2.08e-04 | 1332.6ms/step | 48.0 samples/s
686
+ Step 5590/10001 | Loss: 3.4897 | LR: 2.07e-04 | 1334.2ms/step | 48.0 samples/s
687
+ Step 5600/10001 | Loss: 3.4757 | LR: 2.07e-04 | 1332.8ms/step | 48.0 samples/s
688
+ Step 5610/10001 | Loss: 3.4978 | LR: 2.06e-04 | 1333.8ms/step | 48.0 samples/s
689
+ Step 5620/10001 | Loss: 3.4706 | LR: 2.05e-04 | 1332.8ms/step | 48.0 samples/s
690
+ Step 5630/10001 | Loss: 3.4569 | LR: 2.04e-04 | 1334.4ms/step | 48.0 samples/s
691
+ Step 5640/10001 | Loss: 3.4520 | LR: 2.04e-04 | 1333.3ms/step | 48.0 samples/s
692
+ Step 5650/10001 | Loss: 3.4694 | LR: 2.03e-04 | 1331.6ms/step | 48.1 samples/s
693
+ Step 5660/10001 | Loss: 3.4856 | LR: 2.02e-04 | 1330.5ms/step | 48.1 samples/s
694
+ Step 5670/10001 | Loss: 3.4858 | LR: 2.01e-04 | 1331.4ms/step | 48.1 samples/s
695
+ Step 5680/10001 | Loss: 3.4719 | LR: 2.00e-04 | 1330.2ms/step | 48.1 samples/s
696
+ Step 5690/10001 | Loss: 3.4827 | LR: 2.00e-04 | 1330.0ms/step | 48.1 samples/s
697
+ Step 5700/10001 | Loss: 3.4721 | LR: 1.99e-04 | 1332.0ms/step | 48.0 samples/s
698
+ Step 5710/10001 | Loss: 3.4607 | LR: 1.98e-04 | 1330.9ms/step | 48.1 samples/s
699
+ Step 5720/10001 | Loss: 3.4352 | LR: 1.97e-04 | 1330.8ms/step | 48.1 samples/s
700
+ Step 5730/10001 | Loss: 3.4422 | LR: 1.97e-04 | 1326.3ms/step | 48.3 samples/s
701
+ Step 5740/10001 | Loss: 3.4651 | LR: 1.96e-04 | 1327.9ms/step | 48.2 samples/s
702
+ Step 5750/10001 | Loss: 3.4874 | LR: 1.95e-04 | 1329.9ms/step | 48.1 samples/s
703
+ Validation at step 5750: Loss = 3.4659, Perplexity = 32.00
704
+ Step 5760/10001 | Loss: 3.4545 | LR: 1.94e-04 | 26768.8ms/step | 2.4 samples/s
705
+ Step 5770/10001 | Loss: 3.4743 | LR: 1.93e-04 | 1334.4ms/step | 48.0 samples/s
706
+ Step 5780/10001 | Loss: 3.4798 | LR: 1.93e-04 | 1335.2ms/step | 47.9 samples/s
707
+ Step 5790/10001 | Loss: 3.4694 | LR: 1.92e-04 | 1333.8ms/step | 48.0 samples/s
708
+ Step 5800/10001 | Loss: 3.4227 | LR: 1.91e-04 | 1333.0ms/step | 48.0 samples/s
709
+ Step 5810/10001 | Loss: 3.4735 | LR: 1.90e-04 | 1333.6ms/step | 48.0 samples/s
710
+ Step 5820/10001 | Loss: 3.4358 | LR: 1.90e-04 | 1333.5ms/step | 48.0 samples/s
711
+ Step 5830/10001 | Loss: 3.4682 | LR: 1.89e-04 | 1333.1ms/step | 48.0 samples/s
712
+ Step 5840/10001 | Loss: 3.4716 | LR: 1.88e-04 | 1332.8ms/step | 48.0 samples/s
713
+ Step 5850/10001 | Loss: 3.4725 | LR: 1.87e-04 | 1331.5ms/step | 48.1 samples/s
714
+ Step 5860/10001 | Loss: 3.4540 | LR: 1.86e-04 | 1331.6ms/step | 48.1 samples/s
715
+ Step 5870/10001 | Loss: 3.4533 | LR: 1.86e-04 | 1331.4ms/step | 48.1 samples/s
716
+ Step 5880/10001 | Loss: 3.4698 | LR: 1.85e-04 | 1331.6ms/step | 48.1 samples/s
717
+ Step 5890/10001 | Loss: 3.4518 | LR: 1.84e-04 | 1333.1ms/step | 48.0 samples/s
718
+ Step 5900/10001 | Loss: 3.4427 | LR: 1.83e-04 | 1332.2ms/step | 48.0 samples/s
719
+ Step 5910/10001 | Loss: 3.4315 | LR: 1.83e-04 | 1330.8ms/step | 48.1 samples/s
720
+ Step 5920/10001 | Loss: 3.4411 | LR: 1.82e-04 | 1332.7ms/step | 48.0 samples/s
721
+ Step 5930/10001 | Loss: 3.4618 | LR: 1.81e-04 | 1332.0ms/step | 48.0 samples/s
722
+ Step 5940/10001 | Loss: 3.4792 | LR: 1.80e-04 | 1333.8ms/step | 48.0 samples/s
723
+ Step 5950/10001 | Loss: 3.4532 | LR: 1.80e-04 | 1332.4ms/step | 48.0 samples/s
724
+ Step 5960/10001 | Loss: 3.4601 | LR: 1.79e-04 | 1333.7ms/step | 48.0 samples/s
725
+ Step 5970/10001 | Loss: 3.4483 | LR: 1.78e-04 | 1334.2ms/step | 48.0 samples/s
726
+ Step 5980/10001 | Loss: 3.4586 | LR: 1.77e-04 | 1330.9ms/step | 48.1 samples/s
727
+ Step 5990/10001 | Loss: 3.5065 | LR: 1.77e-04 | 1330.5ms/step | 48.1 samples/s
728
+ Step 6000/10001 | Loss: 3.4334 | LR: 1.76e-04 | 1329.7ms/step | 48.1 samples/s
729
+ Validation at step 6000: Loss = 3.4531, Perplexity = 31.60
730
+ Step 6010/10001 | Loss: 3.4421 | LR: 1.75e-04 | 26776.3ms/step | 2.4 samples/s
731
+ Step 6020/10001 | Loss: 3.4288 | LR: 1.74e-04 | 1334.9ms/step | 47.9 samples/s
732
+ Step 6030/10001 | Loss: 3.4508 | LR: 1.74e-04 | 1334.5ms/step | 48.0 samples/s
733
+ Step 6040/10001 | Loss: 3.4356 | LR: 1.73e-04 | 1334.0ms/step | 48.0 samples/s
734
+ Step 6050/10001 | Loss: 3.4448 | LR: 1.72e-04 | 1335.7ms/step | 47.9 samples/s
735
+ Step 6060/10001 | Loss: 3.4493 | LR: 1.71e-04 | 1334.9ms/step | 47.9 samples/s
736
+ Step 6070/10001 | Loss: 3.4730 | LR: 1.71e-04 | 1336.0ms/step | 47.9 samples/s
737
+ Step 6080/10001 | Loss: 3.4335 | LR: 1.70e-04 | 1334.8ms/step | 47.9 samples/s
738
+ Step 6090/10001 | Loss: 3.4668 | LR: 1.69e-04 | 1335.4ms/step | 47.9 samples/s
739
+ Step 6100/10001 | Loss: 3.4923 | LR: 1.68e-04 | 1333.7ms/step | 48.0 samples/s
740
+ Step 6110/10001 | Loss: 3.4290 | LR: 1.68e-04 | 1335.0ms/step | 47.9 samples/s
741
+ Step 6120/10001 | Loss: 3.4132 | LR: 1.67e-04 | 1334.1ms/step | 48.0 samples/s
742
+ Step 6130/10001 | Loss: 3.4390 | LR: 1.66e-04 | 1333.8ms/step | 48.0 samples/s
743
+ Step 6140/10001 | Loss: 3.4474 | LR: 1.65e-04 | 1332.5ms/step | 48.0 samples/s
744
+ Step 6150/10001 | Loss: 3.4690 | LR: 1.65e-04 | 1333.1ms/step | 48.0 samples/s
745
+ Step 6160/10001 | Loss: 3.4328 | LR: 1.64e-04 | 1332.5ms/step | 48.0 samples/s
746
+ Step 6170/10001 | Loss: 3.4440 | LR: 1.63e-04 | 1331.2ms/step | 48.1 samples/s
747
+ Step 6180/10001 | Loss: 3.4507 | LR: 1.62e-04 | 1332.6ms/step | 48.0 samples/s
748
+ Step 6190/10001 | Loss: 3.4444 | LR: 1.62e-04 | 1330.4ms/step | 48.1 samples/s
749
+ Step 6200/10001 | Loss: 3.4424 | LR: 1.61e-04 | 1414.0ms/step | 45.3 samples/s
750
+ Step 6210/10001 | Loss: 3.4399 | LR: 1.60e-04 | 1332.4ms/step | 48.0 samples/s
751
+ Step 6220/10001 | Loss: 3.4214 | LR: 1.59e-04 | 1333.5ms/step | 48.0 samples/s
752
+ Step 6230/10001 | Loss: 3.4273 | LR: 1.59e-04 | 1334.1ms/step | 48.0 samples/s
753
+ Step 6240/10001 | Loss: 3.4673 | LR: 1.58e-04 | 1333.2ms/step | 48.0 samples/s
754
+ Step 6250/10001 | Loss: 3.4479 | LR: 1.57e-04 | 1333.4ms/step | 48.0 samples/s
755
+ Validation at step 6250: Loss = 3.4428, Perplexity = 31.27
756
+ Step 6260/10001 | Loss: 3.4556 | LR: 1.56e-04 | 26772.7ms/step | 2.4 samples/s
757
+ Step 6270/10001 | Loss: 3.4342 | LR: 1.56e-04 | 1333.0ms/step | 48.0 samples/s
758
+ Step 6280/10001 | Loss: 3.4153 | LR: 1.55e-04 | 1333.1ms/step | 48.0 samples/s
759
+ Step 6290/10001 | Loss: 3.4372 | LR: 1.54e-04 | 1335.2ms/step | 47.9 samples/s
760
+ Step 6300/10001 | Loss: 3.4590 | LR: 1.53e-04 | 1333.5ms/step | 48.0 samples/s
761
+ Step 6310/10001 | Loss: 3.4442 | LR: 1.53e-04 | 1334.4ms/step | 48.0 samples/s
762
+ Step 6320/10001 | Loss: 3.4185 | LR: 1.52e-04 | 1333.4ms/step | 48.0 samples/s
763
+ Step 6330/10001 | Loss: 3.4563 | LR: 1.51e-04 | 1330.5ms/step | 48.1 samples/s
764
+ Step 6340/10001 | Loss: 3.4325 | LR: 1.51e-04 | 1331.3ms/step | 48.1 samples/s
765
+ Step 6350/10001 | Loss: 3.4354 | LR: 1.50e-04 | 1333.7ms/step | 48.0 samples/s
766
+ Step 6360/10001 | Loss: 3.4438 | LR: 1.49e-04 | 1332.6ms/step | 48.0 samples/s
767
+ Step 6370/10001 | Loss: 3.4177 | LR: 1.48e-04 | 1332.1ms/step | 48.0 samples/s
768
+ Step 6380/10001 | Loss: 3.4197 | LR: 1.48e-04 | 1333.5ms/step | 48.0 samples/s
769
+ Step 6390/10001 | Loss: 3.4410 | LR: 1.47e-04 | 1332.1ms/step | 48.0 samples/s
770
+ Step 6400/10001 | Loss: 3.4397 | LR: 1.46e-04 | 1330.7ms/step | 48.1 samples/s
771
+ Step 6410/10001 | Loss: 3.4524 | LR: 1.45e-04 | 1331.8ms/step | 48.1 samples/s
772
+ Step 6420/10001 | Loss: 3.4416 | LR: 1.45e-04 | 1333.6ms/step | 48.0 samples/s
773
+ Step 6430/10001 | Loss: 3.4254 | LR: 1.44e-04 | 1333.2ms/step | 48.0 samples/s
774
+ Step 6440/10001 | Loss: 3.4504 | LR: 1.43e-04 | 1331.3ms/step | 48.1 samples/s
775
+ Step 6450/10001 | Loss: 3.4210 | LR: 1.43e-04 | 1330.8ms/step | 48.1 samples/s
776
+ Step 6460/10001 | Loss: 3.4433 | LR: 1.42e-04 | 1332.4ms/step | 48.0 samples/s
777
+ Step 6470/10001 | Loss: 3.4287 | LR: 1.41e-04 | 1331.4ms/step | 48.1 samples/s
778
+ Step 6480/10001 | Loss: 3.4591 | LR: 1.40e-04 | 1333.7ms/step | 48.0 samples/s
779
+ Step 6490/10001 | Loss: 3.4209 | LR: 1.40e-04 | 1333.5ms/step | 48.0 samples/s
780
+ Step 6500/10001 | Loss: 3.4103 | LR: 1.39e-04 | 1332.9ms/step | 48.0 samples/s
781
+ Validation at step 6500: Loss = 3.4328, Perplexity = 30.96
782
+ Step 6510/10001 | Loss: 3.4197 | LR: 1.38e-04 | 26793.1ms/step | 2.4 samples/s
783
+ Step 6520/10001 | Loss: 3.4212 | LR: 1.38e-04 | 1334.5ms/step | 48.0 samples/s
784
+ Step 6530/10001 | Loss: 3.4527 | LR: 1.37e-04 | 1332.3ms/step | 48.0 samples/s
785
+ Step 6540/10001 | Loss: 3.4418 | LR: 1.36e-04 | 1330.5ms/step | 48.1 samples/s
786
+ Step 6550/10001 | Loss: 3.4643 | LR: 1.35e-04 | 1332.5ms/step | 48.0 samples/s
787
+ Step 6560/10001 | Loss: 3.4325 | LR: 1.35e-04 | 1333.2ms/step | 48.0 samples/s
788
+ Step 6570/10001 | Loss: 3.3960 | LR: 1.34e-04 | 1333.7ms/step | 48.0 samples/s
789
+ Step 6580/10001 | Loss: 3.4418 | LR: 1.33e-04 | 1334.8ms/step | 47.9 samples/s
790
+ Step 6590/10001 | Loss: 3.4291 | LR: 1.33e-04 | 1332.4ms/step | 48.0 samples/s
791
+ Step 6600/10001 | Loss: 3.4220 | LR: 1.32e-04 | 1333.4ms/step | 48.0 samples/s
792
+ Step 6610/10001 | Loss: 3.4270 | LR: 1.31e-04 | 1333.7ms/step | 48.0 samples/s
793
+ Step 6620/10001 | Loss: 3.4028 | LR: 1.31e-04 | 1331.1ms/step | 48.1 samples/s
794
+ Step 6630/10001 | Loss: 3.4127 | LR: 1.30e-04 | 1332.3ms/step | 48.0 samples/s
795
+ Step 6640/10001 | Loss: 3.4136 | LR: 1.29e-04 | 1331.7ms/step | 48.1 samples/s
796
+ Step 6650/10001 | Loss: 3.4324 | LR: 1.28e-04 | 1332.9ms/step | 48.0 samples/s
797
+ Step 6660/10001 | Loss: 3.4390 | LR: 1.28e-04 | 1334.1ms/step | 48.0 samples/s
798
+ Step 6670/10001 | Loss: 3.4450 | LR: 1.27e-04 | 1333.8ms/step | 48.0 samples/s
799
+ Step 6680/10001 | Loss: 3.4241 | LR: 1.26e-04 | 1332.6ms/step | 48.0 samples/s
800
+ Step 6690/10001 | Loss: 3.4336 | LR: 1.26e-04 | 1329.0ms/step | 48.2 samples/s
801
+ Step 6700/10001 | Loss: 3.4121 | LR: 1.25e-04 | 1327.9ms/step | 48.2 samples/s
802
+ Step 6710/10001 | Loss: 3.4368 | LR: 1.24e-04 | 1329.0ms/step | 48.2 samples/s
803
+ Step 6720/10001 | Loss: 3.4266 | LR: 1.24e-04 | 1330.0ms/step | 48.1 samples/s
804
+ Step 6730/10001 | Loss: 3.4112 | LR: 1.23e-04 | 1331.2ms/step | 48.1 samples/s
805
+ Step 6740/10001 | Loss: 3.4050 | LR: 1.22e-04 | 1330.3ms/step | 48.1 samples/s
806
+ Step 6750/10001 | Loss: 3.4247 | LR: 1.22e-04 | 1331.3ms/step | 48.1 samples/s
807
+ Validation at step 6750: Loss = 3.4246, Perplexity = 30.71
808
+ Step 6760/10001 | Loss: 3.4254 | LR: 1.21e-04 | 26787.6ms/step | 2.4 samples/s
809
+ Step 6770/10001 | Loss: 3.4211 | LR: 1.20e-04 | 1335.2ms/step | 47.9 samples/s
810
+ Step 6780/10001 | Loss: 3.4390 | LR: 1.20e-04 | 1332.4ms/step | 48.0 samples/s
811
+ Step 6790/10001 | Loss: 3.4174 | LR: 1.19e-04 | 1335.1ms/step | 47.9 samples/s
812
+ Step 6800/10001 | Loss: 3.3982 | LR: 1.18e-04 | 1333.8ms/step | 48.0 samples/s
813
+ Step 6810/10001 | Loss: 3.4176 | LR: 1.18e-04 | 1334.9ms/step | 47.9 samples/s
814
+ Step 6820/10001 | Loss: 3.4250 | LR: 1.17e-04 | 1333.6ms/step | 48.0 samples/s
815
+ Step 6830/10001 | Loss: 3.3999 | LR: 1.16e-04 | 1333.5ms/step | 48.0 samples/s
816
+ Step 6840/10001 | Loss: 3.4296 | LR: 1.16e-04 | 1333.4ms/step | 48.0 samples/s
817
+ Step 6850/10001 | Loss: 3.4093 | LR: 1.15e-04 | 1333.8ms/step | 48.0 samples/s
818
+ Step 6860/10001 | Loss: 3.4323 | LR: 1.14e-04 | 1333.2ms/step | 48.0 samples/s
819
+ Step 6870/10001 | Loss: 3.4006 | LR: 1.14e-04 | 1336.4ms/step | 47.9 samples/s
820
+ Step 6880/10001 | Loss: 3.4096 | LR: 1.13e-04 | 1336.2ms/step | 47.9 samples/s
821
+ Step 6890/10001 | Loss: 3.4420 | LR: 1.12e-04 | 1335.1ms/step | 47.9 samples/s
822
+ Step 6900/10001 | Loss: 3.4326 | LR: 1.12e-04 | 1335.6ms/step | 47.9 samples/s
823
+ Step 6910/10001 | Loss: 3.4083 | LR: 1.11e-04 | 1333.4ms/step | 48.0 samples/s
824
+ Step 6920/10001 | Loss: 3.4156 | LR: 1.10e-04 | 1333.7ms/step | 48.0 samples/s
825
+ Step 6930/10001 | Loss: 3.4070 | LR: 1.10e-04 | 1416.8ms/step | 45.2 samples/s
826
+ Step 6940/10001 | Loss: 3.4110 | LR: 1.09e-04 | 1332.7ms/step | 48.0 samples/s
827
+ Step 6950/10001 | Loss: 3.4297 | LR: 1.08e-04 | 1332.6ms/step | 48.0 samples/s
828
+ Step 6960/10001 | Loss: 3.4309 | LR: 1.08e-04 | 1333.4ms/step | 48.0 samples/s
829
+ Step 6970/10001 | Loss: 3.4051 | LR: 1.07e-04 | 1334.2ms/step | 48.0 samples/s
830
+ Step 6980/10001 | Loss: 3.4321 | LR: 1.06e-04 | 1333.4ms/step | 48.0 samples/s
831
+ Step 6990/10001 | Loss: 3.4186 | LR: 1.06e-04 | 1333.6ms/step | 48.0 samples/s
832
+ Step 7000/10001 | Loss: 3.3920 | LR: 1.05e-04 | 1333.4ms/step | 48.0 samples/s
833
+ Validation at step 7000: Loss = 3.4175, Perplexity = 30.49
834
+ Step 7010/10001 | Loss: 3.3740 | LR: 1.04e-04 | 26725.3ms/step | 2.4 samples/s
835
+ Step 7020/10001 | Loss: 3.4169 | LR: 1.04e-04 | 1327.5ms/step | 48.2 samples/s
836
+ Step 7030/10001 | Loss: 3.3665 | LR: 1.03e-04 | 1329.4ms/step | 48.1 samples/s
837
+ Step 7040/10001 | Loss: 3.4369 | LR: 1.02e-04 | 1332.2ms/step | 48.0 samples/s
838
+ Step 7050/10001 | Loss: 3.4164 | LR: 1.02e-04 | 1331.6ms/step | 48.1 samples/s
839
+ Step 7060/10001 | Loss: 3.4187 | LR: 1.01e-04 | 1332.6ms/step | 48.0 samples/s
840
+ Step 7070/10001 | Loss: 3.4163 | LR: 1.01e-04 | 1332.7ms/step | 48.0 samples/s
841
+ Step 7080/10001 | Loss: 3.3856 | LR: 9.99e-05 | 1326.3ms/step | 48.3 samples/s
842
+ Step 7090/10001 | Loss: 3.4092 | LR: 9.93e-05 | 1323.7ms/step | 48.3 samples/s
843
+ Step 7100/10001 | Loss: 3.4024 | LR: 9.86e-05 | 1324.3ms/step | 48.3 samples/s
844
+ Step 7110/10001 | Loss: 3.3911 | LR: 9.80e-05 | 1325.2ms/step | 48.3 samples/s
845
+ Step 7120/10001 | Loss: 3.4002 | LR: 9.74e-05 | 1328.1ms/step | 48.2 samples/s
846
+ Step 7130/10001 | Loss: 3.4200 | LR: 9.68e-05 | 1327.8ms/step | 48.2 samples/s
847
+ Step 7140/10001 | Loss: 3.4193 | LR: 9.61e-05 | 1329.4ms/step | 48.1 samples/s
848
+ Step 7150/10001 | Loss: 3.4246 | LR: 9.55e-05 | 1329.0ms/step | 48.2 samples/s
849
+ Step 7160/10001 | Loss: 3.4354 | LR: 9.49e-05 | 1328.5ms/step | 48.2 samples/s
850
+ Step 7170/10001 | Loss: 3.3975 | LR: 9.43e-05 | 1327.4ms/step | 48.2 samples/s
851
+ Step 7180/10001 | Loss: 3.3963 | LR: 9.36e-05 | 1327.5ms/step | 48.2 samples/s
852
+ Step 7190/10001 | Loss: 3.4130 | LR: 9.30e-05 | 1327.6ms/step | 48.2 samples/s
853
+ Step 7200/10001 | Loss: 3.4187 | LR: 9.24e-05 | 1327.9ms/step | 48.2 samples/s
854
+ Step 7210/10001 | Loss: 3.4227 | LR: 9.18e-05 | 1326.5ms/step | 48.2 samples/s
855
+ Step 7220/10001 | Loss: 3.4070 | LR: 9.12e-05 | 1326.3ms/step | 48.3 samples/s
856
+ Step 7230/10001 | Loss: 3.4147 | LR: 9.06e-05 | 1325.3ms/step | 48.3 samples/s
857
+ Step 7240/10001 | Loss: 3.3873 | LR: 9.00e-05 | 1328.7ms/step | 48.2 samples/s
858
+ Step 7250/10001 | Loss: 3.4215 | LR: 8.93e-05 | 1326.2ms/step | 48.3 samples/s
859
+ Validation at step 7250: Loss = 3.4126, Perplexity = 30.34
860
+ Step 7260/10001 | Loss: 3.3824 | LR: 8.87e-05 | 26670.1ms/step | 2.4 samples/s
861
+ Step 7270/10001 | Loss: 3.4261 | LR: 8.81e-05 | 1326.9ms/step | 48.2 samples/s
862
+ Step 7280/10001 | Loss: 3.4349 | LR: 8.75e-05 | 1327.4ms/step | 48.2 samples/s
863
+ Step 7290/10001 | Loss: 3.4071 | LR: 8.69e-05 | 1326.9ms/step | 48.2 samples/s
864
+ Step 7300/10001 | Loss: 3.4122 | LR: 8.63e-05 | 1328.5ms/step | 48.2 samples/s
865
+ Step 7310/10001 | Loss: 3.4000 | LR: 8.57e-05 | 1333.5ms/step | 48.0 samples/s
866
+ Step 7320/10001 | Loss: 3.4080 | LR: 8.51e-05 | 1332.9ms/step | 48.0 samples/s
867
+ Step 7330/10001 | Loss: 3.4220 | LR: 8.45e-05 | 1328.0ms/step | 48.2 samples/s
868
+ Step 7340/10001 | Loss: 3.3862 | LR: 8.39e-05 | 1327.6ms/step | 48.2 samples/s
869
+ Step 7350/10001 | Loss: 3.4192 | LR: 8.34e-05 | 1329.1ms/step | 48.2 samples/s
870
+ Step 7360/10001 | Loss: 3.4211 | LR: 8.28e-05 | 1328.7ms/step | 48.2 samples/s
871
+ Step 7370/10001 | Loss: 3.4176 | LR: 8.22e-05 | 1332.3ms/step | 48.0 samples/s
872
+ Step 7380/10001 | Loss: 3.4285 | LR: 8.16e-05 | 1330.4ms/step | 48.1 samples/s
873
+ Step 7390/10001 | Loss: 3.3988 | LR: 8.10e-05 | 1328.6ms/step | 48.2 samples/s
874
+ Step 7400/10001 | Loss: 3.4084 | LR: 8.04e-05 | 1328.1ms/step | 48.2 samples/s
875
+ Step 7410/10001 | Loss: 3.4257 | LR: 7.98e-05 | 1326.9ms/step | 48.2 samples/s
876
+ Step 7420/10001 | Loss: 3.3903 | LR: 7.93e-05 | 1327.5ms/step | 48.2 samples/s
877
+ Step 7430/10001 | Loss: 3.4163 | LR: 7.87e-05 | 1327.8ms/step | 48.2 samples/s
878
+ Step 7440/10001 | Loss: 3.3936 | LR: 7.81e-05 | 1325.9ms/step | 48.3 samples/s
879
+ Step 7450/10001 | Loss: 3.4141 | LR: 7.75e-05 | 1328.3ms/step | 48.2 samples/s
880
+ Step 7460/10001 | Loss: 3.4058 | LR: 7.69e-05 | 1330.9ms/step | 48.1 samples/s
881
+ Step 7470/10001 | Loss: 3.3838 | LR: 7.64e-05 | 1330.5ms/step | 48.1 samples/s
882
+ Step 7480/10001 | Loss: 3.4000 | LR: 7.58e-05 | 1327.8ms/step | 48.2 samples/s
883
+ Step 7490/10001 | Loss: 3.3901 | LR: 7.52e-05 | 1328.5ms/step | 48.2 samples/s
884
+ Step 7500/10001 | Loss: 3.4342 | LR: 7.47e-05 | 1329.3ms/step | 48.1 samples/s
885
+ Validation at step 7500: Loss = 3.4084, Perplexity = 30.22
886
+ Saved checkpoint to checkpoints/run_20260430_021312/checkpoint-7500
887
+ Step 7510/10001 | Loss: 3.4247 | LR: 7.41e-05 | 27199.4ms/step | 2.4 samples/s
888
+ Step 7520/10001 | Loss: 3.3754 | LR: 7.35e-05 | 1324.1ms/step | 48.3 samples/s
889
+ Step 7530/10001 | Loss: 3.4232 | LR: 7.30e-05 | 1323.4ms/step | 48.4 samples/s
890
+ Step 7540/10001 | Loss: 3.4074 | LR: 7.24e-05 | 1325.4ms/step | 48.3 samples/s
891
+ Step 7550/10001 | Loss: 3.3914 | LR: 7.19e-05 | 1328.2ms/step | 48.2 samples/s
892
+ Step 7560/10001 | Loss: 3.3972 | LR: 7.13e-05 | 1326.3ms/step | 48.3 samples/s
893
+ Step 7570/10001 | Loss: 3.3957 | LR: 7.08e-05 | 1327.7ms/step | 48.2 samples/s
894
+ Step 7580/10001 | Loss: 3.3820 | LR: 7.02e-05 | 1328.2ms/step | 48.2 samples/s
895
+ Step 7590/10001 | Loss: 3.4145 | LR: 6.97e-05 | 1328.8ms/step | 48.2 samples/s
896
+ Step 7600/10001 | Loss: 3.4238 | LR: 6.91e-05 | 1332.0ms/step | 48.0 samples/s
897
+ Step 7610/10001 | Loss: 3.3798 | LR: 6.86e-05 | 1331.7ms/step | 48.1 samples/s
898
+ Step 7620/10001 | Loss: 3.4363 | LR: 6.80e-05 | 1330.6ms/step | 48.1 samples/s
899
+ Step 7630/10001 | Loss: 3.4070 | LR: 6.75e-05 | 1329.4ms/step | 48.1 samples/s
900
+ Step 7640/10001 | Loss: 3.3944 | LR: 6.69e-05 | 1331.7ms/step | 48.1 samples/s
901
+ Step 7650/10001 | Loss: 3.4163 | LR: 6.64e-05 | 1418.5ms/step | 45.1 samples/s
902
+ Step 7660/10001 | Loss: 3.3950 | LR: 6.59e-05 | 1326.3ms/step | 48.3 samples/s
903
+ Step 7670/10001 | Loss: 3.3959 | LR: 6.53e-05 | 1329.6ms/step | 48.1 samples/s
904
+ Step 7680/10001 | Loss: 3.4175 | LR: 6.48e-05 | 1329.2ms/step | 48.2 samples/s
905
+ Step 7690/10001 | Loss: 3.3984 | LR: 6.43e-05 | 1327.4ms/step | 48.2 samples/s
906
+ Step 7700/10001 | Loss: 3.3841 | LR: 6.37e-05 | 1327.0ms/step | 48.2 samples/s
907
+ Step 7710/10001 | Loss: 3.4037 | LR: 6.32e-05 | 1328.3ms/step | 48.2 samples/s
908
+ Step 7720/10001 | Loss: 3.4038 | LR: 6.27e-05 | 1328.3ms/step | 48.2 samples/s
909
+ Step 7730/10001 | Loss: 3.4036 | LR: 6.21e-05 | 1329.8ms/step | 48.1 samples/s
910
+ Step 7740/10001 | Loss: 3.3908 | LR: 6.16e-05 | 1328.9ms/step | 48.2 samples/s
911
+ Step 7750/10001 | Loss: 3.3917 | LR: 6.11e-05 | 1328.2ms/step | 48.2 samples/s
912
+ Validation at step 7750: Loss = 3.4051, Perplexity = 30.12
913
+ Step 7760/10001 | Loss: 3.4268 | LR: 6.06e-05 | 26711.0ms/step | 2.4 samples/s
914
+ Step 7770/10001 | Loss: 3.4049 | LR: 6.01e-05 | 1328.1ms/step | 48.2 samples/s
915
+ Step 7780/10001 | Loss: 3.3888 | LR: 5.96e-05 | 1326.7ms/step | 48.2 samples/s
916
+ Step 7790/10001 | Loss: 3.3964 | LR: 5.90e-05 | 1328.8ms/step | 48.2 samples/s
917
+ Step 7800/10001 | Loss: 3.3817 | LR: 5.85e-05 | 1326.3ms/step | 48.3 samples/s
918
+ Step 7810/10001 | Loss: 3.4161 | LR: 5.80e-05 | 1326.4ms/step | 48.3 samples/s
919
+ Step 7820/10001 | Loss: 3.4104 | LR: 5.75e-05 | 1327.1ms/step | 48.2 samples/s
920
+ Step 7830/10001 | Loss: 3.3836 | LR: 5.70e-05 | 1326.0ms/step | 48.3 samples/s
921
+ Step 7840/10001 | Loss: 3.4041 | LR: 5.65e-05 | 1326.1ms/step | 48.3 samples/s
922
+ Step 7850/10001 | Loss: 3.3843 | LR: 5.60e-05 | 1328.5ms/step | 48.2 samples/s
923
+ Step 7860/10001 | Loss: 3.3880 | LR: 5.55e-05 | 1329.8ms/step | 48.1 samples/s
924
+ Step 7870/10001 | Loss: 3.3941 | LR: 5.50e-05 | 1327.3ms/step | 48.2 samples/s
925
+ Step 7880/10001 | Loss: 3.3923 | LR: 5.45e-05 | 1329.6ms/step | 48.1 samples/s
926
+ Step 7890/10001 | Loss: 3.3659 | LR: 5.40e-05 | 1330.6ms/step | 48.1 samples/s
927
+ Step 7900/10001 | Loss: 3.4219 | LR: 5.35e-05 | 1328.2ms/step | 48.2 samples/s
928
+ Step 7910/10001 | Loss: 3.3754 | LR: 5.30e-05 | 1328.9ms/step | 48.2 samples/s
929
+ Step 7920/10001 | Loss: 3.3888 | LR: 5.25e-05 | 1324.9ms/step | 48.3 samples/s
930
+ Step 7930/10001 | Loss: 3.4140 | LR: 5.21e-05 | 1324.2ms/step | 48.3 samples/s
931
+ Step 7940/10001 | Loss: 3.4067 | LR: 5.16e-05 | 1331.0ms/step | 48.1 samples/s
932
+ Step 7950/10001 | Loss: 3.4028 | LR: 5.11e-05 | 1328.2ms/step | 48.2 samples/s
933
+ Step 7960/10001 | Loss: 3.3858 | LR: 5.06e-05 | 1325.9ms/step | 48.3 samples/s
934
+ Step 7970/10001 | Loss: 3.3936 | LR: 5.01e-05 | 1323.0ms/step | 48.4 samples/s
935
+ Step 7980/10001 | Loss: 3.4035 | LR: 4.97e-05 | 1327.1ms/step | 48.2 samples/s
936
+ Step 7990/10001 | Loss: 3.4049 | LR: 4.92e-05 | 1328.4ms/step | 48.2 samples/s
937
+ Step 8000/10001 | Loss: 3.4058 | LR: 4.87e-05 | 1329.8ms/step | 48.1 samples/s
938
+ Validation at step 8000: Loss = 3.4029, Perplexity = 30.05
939
+ Step 8010/10001 | Loss: 3.3876 | LR: 4.83e-05 | 26704.2ms/step | 2.4 samples/s
940
+ Step 8020/10001 | Loss: 3.3979 | LR: 4.78e-05 | 1328.0ms/step | 48.2 samples/s
941
+ Step 8030/10001 | Loss: 3.4034 | LR: 4.73e-05 | 1326.9ms/step | 48.2 samples/s
942
+ Step 8040/10001 | Loss: 3.3692 | LR: 4.69e-05 | 1331.1ms/step | 48.1 samples/s
943
+ Step 8050/10001 | Loss: 3.4111 | LR: 4.64e-05 | 1331.7ms/step | 48.1 samples/s
944
+ Step 8060/10001 | Loss: 3.3965 | LR: 4.59e-05 | 1327.9ms/step | 48.2 samples/s
945
+ Step 8070/10001 | Loss: 3.4013 | LR: 4.55e-05 | 1328.5ms/step | 48.2 samples/s
946
+ Step 8080/10001 | Loss: 3.3922 | LR: 4.50e-05 | 1326.2ms/step | 48.3 samples/s
947
+ Step 8090/10001 | Loss: 3.3675 | LR: 4.46e-05 | 1325.8ms/step | 48.3 samples/s
948
+ Step 8100/10001 | Loss: 3.4190 | LR: 4.41e-05 | 1325.0ms/step | 48.3 samples/s
949
+ Step 8110/10001 | Loss: 3.3877 | LR: 4.37e-05 | 1327.5ms/step | 48.2 samples/s
950
+ Step 8120/10001 | Loss: 3.4209 | LR: 4.32e-05 | 1331.6ms/step | 48.1 samples/s
951
+ Step 8130/10001 | Loss: 3.3873 | LR: 4.28e-05 | 1330.2ms/step | 48.1 samples/s
952
+ Step 8140/10001 | Loss: 3.3919 | LR: 4.23e-05 | 1333.4ms/step | 48.0 samples/s
953
+ Step 8150/10001 | Loss: 3.3988 | LR: 4.19e-05 | 1332.2ms/step | 48.0 samples/s
954
+ Step 8160/10001 | Loss: 3.4289 | LR: 4.15e-05 | 1326.7ms/step | 48.2 samples/s
955
+ Step 8170/10001 | Loss: 3.4039 | LR: 4.10e-05 | 1324.9ms/step | 48.3 samples/s
956
+ Step 8180/10001 | Loss: 3.3975 | LR: 4.06e-05 | 1325.9ms/step | 48.3 samples/s
957
+ Step 8190/10001 | Loss: 3.4207 | LR: 4.02e-05 | 1329.4ms/step | 48.1 samples/s
958
+ Step 8200/10001 | Loss: 3.4061 | LR: 3.97e-05 | 1327.1ms/step | 48.2 samples/s
959
+ Step 8210/10001 | Loss: 3.3947 | LR: 3.93e-05 | 1326.6ms/step | 48.2 samples/s
960
+ Step 8220/10001 | Loss: 3.3898 | LR: 3.89e-05 | 1325.2ms/step | 48.3 samples/s
961
+ Step 8230/10001 | Loss: 3.3881 | LR: 3.84e-05 | 1329.4ms/step | 48.1 samples/s
962
+ Step 8240/10001 | Loss: 3.3950 | LR: 3.80e-05 | 1331.7ms/step | 48.1 samples/s
963
+ Step 8250/10001 | Loss: 3.4100 | LR: 3.76e-05 | 1328.2ms/step | 48.2 samples/s
964
+ Validation at step 8250: Loss = 3.4013, Perplexity = 30.00
965
+ Step 8260/10001 | Loss: 3.3703 | LR: 3.72e-05 | 26682.9ms/step | 2.4 samples/s
966
+ Step 8270/10001 | Loss: 3.3776 | LR: 3.68e-05 | 1329.9ms/step | 48.1 samples/s
967
+ Step 8280/10001 | Loss: 3.4062 | LR: 3.64e-05 | 1327.9ms/step | 48.2 samples/s
968
+ Step 8290/10001 | Loss: 3.3712 | LR: 3.59e-05 | 1324.6ms/step | 48.3 samples/s
969
+ Step 8300/10001 | Loss: 3.4139 | LR: 3.55e-05 | 1330.5ms/step | 48.1 samples/s
970
+ Step 8310/10001 | Loss: 3.3785 | LR: 3.51e-05 | 1327.6ms/step | 48.2 samples/s
971
+ Step 8320/10001 | Loss: 3.4026 | LR: 3.47e-05 | 1327.1ms/step | 48.2 samples/s
972
+ Step 8330/10001 | Loss: 3.3957 | LR: 3.43e-05 | 1327.1ms/step | 48.2 samples/s
973
+ Step 8340/10001 | Loss: 3.4167 | LR: 3.39e-05 | 1324.8ms/step | 48.3 samples/s
974
+ Step 8350/10001 | Loss: 3.4154 | LR: 3.35e-05 | 1325.3ms/step | 48.3 samples/s
975
+ Step 8360/10001 | Loss: 3.4162 | LR: 3.31e-05 | 1324.4ms/step | 48.3 samples/s
976
+ Step 8370/10001 | Loss: 3.4042 | LR: 3.27e-05 | 1323.8ms/step | 48.3 samples/s
977
+ Step 8380/10001 | Loss: 3.3846 | LR: 3.23e-05 | 1407.9ms/step | 45.5 samples/s
978
+ Step 8390/10001 | Loss: 3.4020 | LR: 3.20e-05 | 1328.1ms/step | 48.2 samples/s
979
+ Step 8400/10001 | Loss: 3.3986 | LR: 3.16e-05 | 1330.5ms/step | 48.1 samples/s
980
+ Step 8410/10001 | Loss: 3.4204 | LR: 3.12e-05 | 1330.7ms/step | 48.1 samples/s
981
+ Step 8420/10001 | Loss: 3.3641 | LR: 3.08e-05 | 1325.5ms/step | 48.3 samples/s
982
+ Step 8430/10001 | Loss: 3.4123 | LR: 3.04e-05 | 1325.3ms/step | 48.3 samples/s
983
+ Step 8440/10001 | Loss: 3.4033 | LR: 3.00e-05 | 1324.7ms/step | 48.3 samples/s
984
+ Step 8450/10001 | Loss: 3.4202 | LR: 2.97e-05 | 1327.0ms/step | 48.2 samples/s
985
+ Step 8460/10001 | Loss: 3.4154 | LR: 2.93e-05 | 1327.6ms/step | 48.2 samples/s
986
+ Step 8470/10001 | Loss: 3.3952 | LR: 2.89e-05 | 1328.6ms/step | 48.2 samples/s
987
+ Step 8480/10001 | Loss: 3.4044 | LR: 2.86e-05 | 1327.9ms/step | 48.2 samples/s
988
+ Step 8490/10001 | Loss: 3.4008 | LR: 2.82e-05 | 1326.9ms/step | 48.2 samples/s
989
+ Step 8500/10001 | Loss: 3.4234 | LR: 2.78e-05 | 1327.4ms/step | 48.2 samples/s
990
+ Validation at step 8500: Loss = 3.4006, Perplexity = 29.98
991
+ Step 8510/10001 | Loss: 3.4089 | LR: 2.75e-05 | 26705.7ms/step | 2.4 samples/s
992
+ Step 8520/10001 | Loss: 3.4145 | LR: 2.71e-05 | 1332.3ms/step | 48.0 samples/s
993
+ Step 8530/10001 | Loss: 3.4101 | LR: 2.67e-05 | 1331.3ms/step | 48.1 samples/s
994
+ Step 8540/10001 | Loss: 3.3780 | LR: 2.64e-05 | 1333.2ms/step | 48.0 samples/s
995
+ Step 8550/10001 | Loss: 3.4038 | LR: 2.60e-05 | 1333.3ms/step | 48.0 samples/s
996
+ Step 8560/10001 | Loss: 3.3900 | LR: 2.57e-05 | 1329.0ms/step | 48.2 samples/s
997
+ Step 8570/10001 | Loss: 3.3916 | LR: 2.53e-05 | 1332.2ms/step | 48.0 samples/s
998
+ Step 8580/10001 | Loss: 3.3681 | LR: 2.50e-05 | 1331.6ms/step | 48.1 samples/s
999
+ Step 8590/10001 | Loss: 3.3918 | LR: 2.46e-05 | 1330.9ms/step | 48.1 samples/s
1000
+ Step 8600/10001 | Loss: 3.3878 | LR: 2.43e-05 | 1328.8ms/step | 48.2 samples/s
1001
+ Step 8610/10001 | Loss: 3.4172 | LR: 2.40e-05 | 1330.1ms/step | 48.1 samples/s
1002
+ Step 8620/10001 | Loss: 3.3925 | LR: 2.36e-05 | 1334.8ms/step | 47.9 samples/s
1003
+ Step 8630/10001 | Loss: 3.3877 | LR: 2.33e-05 | 1331.5ms/step | 48.1 samples/s
1004
+ Step 8640/10001 | Loss: 3.4051 | LR: 2.30e-05 | 1333.2ms/step | 48.0 samples/s
1005
+ Step 8650/10001 | Loss: 3.4007 | LR: 2.26e-05 | 1333.1ms/step | 48.0 samples/s
1006
+ Step 8660/10001 | Loss: 3.4138 | LR: 2.23e-05 | 1327.0ms/step | 48.2 samples/s
1007
+ Step 8670/10001 | Loss: 3.3822 | LR: 2.20e-05 | 1328.5ms/step | 48.2 samples/s
1008
+ Step 8680/10001 | Loss: 3.4050 | LR: 2.16e-05 | 1327.8ms/step | 48.2 samples/s
1009
+ Step 8690/10001 | Loss: 3.4222 | LR: 2.13e-05 | 1328.2ms/step | 48.2 samples/s
1010
+ Step 8700/10001 | Loss: 3.3684 | LR: 2.10e-05 | 1329.0ms/step | 48.2 samples/s
1011
+ Step 8710/10001 | Loss: 3.3836 | LR: 2.07e-05 | 1327.7ms/step | 48.2 samples/s
1012
+ Step 8720/10001 | Loss: 3.3983 | LR: 2.04e-05 | 1329.9ms/step | 48.1 samples/s
1013
+ Step 8730/10001 | Loss: 3.4051 | LR: 2.01e-05 | 1326.6ms/step | 48.2 samples/s
1014
+ Step 8740/10001 | Loss: 3.3920 | LR: 1.97e-05 | 1329.8ms/step | 48.1 samples/s
1015
+ Step 8750/10001 | Loss: 3.3974 | LR: 1.94e-05 | 1328.6ms/step | 48.2 samples/s
1016
+ Validation at step 8750: Loss = 3.4000, Perplexity = 29.96
1017
+ Step 8760/10001 | Loss: 3.4127 | LR: 1.91e-05 | 26752.1ms/step | 2.4 samples/s
1018
+ Step 8770/10001 | Loss: 3.4185 | LR: 1.88e-05 | 1333.6ms/step | 48.0 samples/s
1019
+ Step 8780/10001 | Loss: 3.3983 | LR: 1.85e-05 | 1329.4ms/step | 48.1 samples/s
1020
+ Step 8790/10001 | Loss: 3.4124 | LR: 1.82e-05 | 1327.0ms/step | 48.2 samples/s
1021
+ Step 8800/10001 | Loss: 3.4134 | LR: 1.79e-05 | 1327.6ms/step | 48.2 samples/s
1022
+ Step 8810/10001 | Loss: 3.3824 | LR: 1.76e-05 | 1327.7ms/step | 48.2 samples/s
1023
+ Step 8820/10001 | Loss: 3.3851 | LR: 1.73e-05 | 1327.0ms/step | 48.2 samples/s
1024
+ Step 8830/10001 | Loss: 3.3935 | LR: 1.71e-05 | 1327.3ms/step | 48.2 samples/s
1025
+ Step 8840/10001 | Loss: 3.4231 | LR: 1.68e-05 | 1327.2ms/step | 48.2 samples/s
1026
+ Step 8850/10001 | Loss: 3.4093 | LR: 1.65e-05 | 1334.5ms/step | 48.0 samples/s
1027
+ Step 8860/10001 | Loss: 3.3888 | LR: 1.62e-05 | 1332.1ms/step | 48.0 samples/s
1028
+ Step 8870/10001 | Loss: 3.3815 | LR: 1.59e-05 | 1330.7ms/step | 48.1 samples/s
1029
+ Step 8880/10001 | Loss: 3.3904 | LR: 1.56e-05 | 1333.1ms/step | 48.0 samples/s
1030
+ Step 8890/10001 | Loss: 3.4091 | LR: 1.54e-05 | 1328.0ms/step | 48.2 samples/s
1031
+ Step 8900/10001 | Loss: 3.3866 | LR: 1.51e-05 | 1327.7ms/step | 48.2 samples/s
1032
+ Step 8910/10001 | Loss: 3.4014 | LR: 1.48e-05 | 1326.8ms/step | 48.2 samples/s
1033
+ Step 8920/10001 | Loss: 3.3931 | LR: 1.46e-05 | 1330.7ms/step | 48.1 samples/s
1034
+ Step 8930/10001 | Loss: 3.4024 | LR: 1.43e-05 | 1333.0ms/step | 48.0 samples/s
1035
+ Step 8940/10001 | Loss: 3.3791 | LR: 1.40e-05 | 1333.3ms/step | 48.0 samples/s
1036
+ Step 8950/10001 | Loss: 3.3958 | LR: 1.38e-05 | 1329.6ms/step | 48.1 samples/s
1037
+ Step 8960/10001 | Loss: 3.3925 | LR: 1.35e-05 | 1329.1ms/step | 48.2 samples/s
1038
+ Step 8970/10001 | Loss: 3.3925 | LR: 1.33e-05 | 1329.5ms/step | 48.1 samples/s
1039
+ Step 8980/10001 | Loss: 3.3859 | LR: 1.30e-05 | 1326.8ms/step | 48.2 samples/s
1040
+ Step 8990/10001 | Loss: 3.4167 | LR: 1.28e-05 | 1327.8ms/step | 48.2 samples/s
1041
+ Step 9000/10001 | Loss: 3.4172 | LR: 1.25e-05 | 1328.5ms/step | 48.2 samples/s
1042
+ Validation at step 9000: Loss = 3.3998, Perplexity = 29.96
1043
+ Step 9010/10001 | Loss: 3.3921 | LR: 1.23e-05 | 26763.7ms/step | 2.4 samples/s
1044
+ Step 9020/10001 | Loss: 3.4195 | LR: 1.20e-05 | 1327.5ms/step | 48.2 samples/s
1045
+ Step 9030/10001 | Loss: 3.4112 | LR: 1.18e-05 | 1328.8ms/step | 48.2 samples/s
1046
+ Step 9040/10001 | Loss: 3.3904 | LR: 1.15e-05 | 1331.1ms/step | 48.1 samples/s
1047
+ Step 9050/10001 | Loss: 3.4004 | LR: 1.13e-05 | 1329.8ms/step | 48.1 samples/s
1048
+ Step 9060/10001 | Loss: 3.3976 | LR: 1.11e-05 | 1326.8ms/step | 48.2 samples/s
1049
+ Step 9070/10001 | Loss: 3.4029 | LR: 1.08e-05 | 1324.7ms/step | 48.3 samples/s
1050
+ Step 9080/10001 | Loss: 3.4125 | LR: 1.06e-05 | 1330.1ms/step | 48.1 samples/s
1051
+ Step 9090/10001 | Loss: 3.3900 | LR: 1.04e-05 | 1329.4ms/step | 48.1 samples/s
1052
+ Step 9100/10001 | Loss: 3.3945 | LR: 1.01e-05 | 1329.0ms/step | 48.2 samples/s
1053
+ Step 9110/10001 | Loss: 3.4011 | LR: 9.92e-06 | 1413.5ms/step | 45.3 samples/s
1054
+ Step 9120/10001 | Loss: 3.3852 | LR: 9.70e-06 | 1324.8ms/step | 48.3 samples/s
1055
+ Step 9130/10001 | Loss: 3.3973 | LR: 9.49e-06 | 1328.6ms/step | 48.2 samples/s
1056
+ Step 9140/10001 | Loss: 3.4096 | LR: 9.27e-06 | 1330.0ms/step | 48.1 samples/s
1057
+ Step 9150/10001 | Loss: 3.3624 | LR: 9.06e-06 | 1326.0ms/step | 48.3 samples/s
1058
+ Step 9160/10001 | Loss: 3.3797 | LR: 8.85e-06 | 1329.9ms/step | 48.1 samples/s
1059
+ Step 9170/10001 | Loss: 3.3869 | LR: 8.64e-06 | 1329.4ms/step | 48.1 samples/s
1060
+ Step 9180/10001 | Loss: 3.3911 | LR: 8.43e-06 | 1329.4ms/step | 48.1 samples/s
1061
+ Step 9190/10001 | Loss: 3.4098 | LR: 8.23e-06 | 1329.8ms/step | 48.1 samples/s
1062
+ Step 9200/10001 | Loss: 3.3780 | LR: 8.03e-06 | 1326.5ms/step | 48.2 samples/s
1063
+ Step 9210/10001 | Loss: 3.4062 | LR: 7.83e-06 | 1327.9ms/step | 48.2 samples/s
1064
+ Step 9220/10001 | Loss: 3.4286 | LR: 7.64e-06 | 1329.4ms/step | 48.1 samples/s
1065
+ Step 9230/10001 | Loss: 3.3823 | LR: 7.44e-06 | 1329.2ms/step | 48.1 samples/s
1066
+ Step 9240/10001 | Loss: 3.4109 | LR: 7.25e-06 | 1329.7ms/step | 48.1 samples/s
1067
+ Step 9250/10001 | Loss: 3.4104 | LR: 7.06e-06 | 1331.6ms/step | 48.1 samples/s
1068
+ Validation at step 9250: Loss = 3.3997, Perplexity = 29.95
1069
+ Step 9260/10001 | Loss: 3.4190 | LR: 6.88e-06 | 26739.8ms/step | 2.4 samples/s
1070
+ Step 9270/10001 | Loss: 3.3909 | LR: 6.69e-06 | 1329.9ms/step | 48.1 samples/s
1071
+ Step 9280/10001 | Loss: 3.4118 | LR: 6.51e-06 | 1329.0ms/step | 48.2 samples/s
1072
+ Step 9290/10001 | Loss: 3.3902 | LR: 6.34e-06 | 1325.5ms/step | 48.3 samples/s
1073
+ Step 9300/10001 | Loss: 3.4130 | LR: 6.16e-06 | 1336.0ms/step | 47.9 samples/s
1074
+ Step 9310/10001 | Loss: 3.3636 | LR: 5.99e-06 | 1329.5ms/step | 48.1 samples/s
1075
+ Step 9320/10001 | Loss: 3.3507 | LR: 5.81e-06 | 1325.9ms/step | 48.3 samples/s
1076
+ Step 9330/10001 | Loss: 3.4009 | LR: 5.64e-06 | 1327.5ms/step | 48.2 samples/s
1077
+ Step 9340/10001 | Loss: 3.3831 | LR: 5.48e-06 | 1329.1ms/step | 48.2 samples/s
1078
+ Step 9350/10001 | Loss: 3.4181 | LR: 5.31e-06 | 1329.6ms/step | 48.1 samples/s
1079
+ Step 9360/10001 | Loss: 3.4007 | LR: 5.15e-06 | 1325.7ms/step | 48.3 samples/s
1080
+ Step 9370/10001 | Loss: 3.4378 | LR: 4.99e-06 | 1327.9ms/step | 48.2 samples/s
1081
+ Step 9380/10001 | Loss: 3.3995 | LR: 4.84e-06 | 1330.3ms/step | 48.1 samples/s
1082
+ Step 9390/10001 | Loss: 3.4167 | LR: 4.68e-06 | 1331.2ms/step | 48.1 samples/s
1083
+ Step 9400/10001 | Loss: 3.3941 | LR: 4.53e-06 | 1330.8ms/step | 48.1 samples/s
1084
+ Step 9410/10001 | Loss: 3.4278 | LR: 4.38e-06 | 1329.1ms/step | 48.2 samples/s
1085
+ Step 9420/10001 | Loss: 3.3838 | LR: 4.24e-06 | 1327.1ms/step | 48.2 samples/s
1086
+ Step 9430/10001 | Loss: 3.3970 | LR: 4.09e-06 | 1326.6ms/step | 48.2 samples/s
1087
+ Step 9440/10001 | Loss: 3.4018 | LR: 3.95e-06 | 1329.6ms/step | 48.1 samples/s
1088
+ Step 9450/10001 | Loss: 3.3956 | LR: 3.81e-06 | 1330.6ms/step | 48.1 samples/s
1089
+ Step 9460/10001 | Loss: 3.4210 | LR: 3.67e-06 | 1329.3ms/step | 48.1 samples/s
1090
+ Step 9470/10001 | Loss: 3.3766 | LR: 3.54e-06 | 1327.3ms/step | 48.2 samples/s
1091
+ Step 9480/10001 | Loss: 3.3988 | LR: 3.41e-06 | 1326.8ms/step | 48.2 samples/s
1092
+ Step 9490/10001 | Loss: 3.4177 | LR: 3.28e-06 | 1337.2ms/step | 47.9 samples/s
1093
+ Step 9500/10001 | Loss: 3.4054 | LR: 3.15e-06 | 1331.4ms/step | 48.1 samples/s
1094
+ Validation at step 9500: Loss = 3.3997, Perplexity = 29.95
1095
+ Step 9510/10001 | Loss: 3.4153 | LR: 3.03e-06 | 26780.2ms/step | 2.4 samples/s
1096
+ Step 9520/10001 | Loss: 3.3939 | LR: 2.91e-06 | 1327.6ms/step | 48.2 samples/s
1097
+ Step 9530/10001 | Loss: 3.3859 | LR: 2.79e-06 | 1331.6ms/step | 48.1 samples/s
1098
+ Step 9540/10001 | Loss: 3.4004 | LR: 2.67e-06 | 1330.3ms/step | 48.1 samples/s
1099
+ Step 9550/10001 | Loss: 3.4326 | LR: 2.56e-06 | 1333.1ms/step | 48.0 samples/s
1100
+ Step 9560/10001 | Loss: 3.3917 | LR: 2.44e-06 | 1334.8ms/step | 47.9 samples/s
1101
+ Step 9570/10001 | Loss: 3.3906 | LR: 2.33e-06 | 1332.2ms/step | 48.0 samples/s
1102
+ Step 9580/10001 | Loss: 3.3876 | LR: 2.23e-06 | 1330.6ms/step | 48.1 samples/s
1103
+ Step 9590/10001 | Loss: 3.4186 | LR: 2.12e-06 | 1332.0ms/step | 48.0 samples/s
1104
+ Step 9600/10001 | Loss: 3.3999 | LR: 2.02e-06 | 1331.3ms/step | 48.1 samples/s
1105
+ Step 9610/10001 | Loss: 3.3764 | LR: 1.92e-06 | 1328.9ms/step | 48.2 samples/s
1106
+ Step 9620/10001 | Loss: 3.3915 | LR: 1.82e-06 | 1336.1ms/step | 47.9 samples/s
1107
+ Step 9630/10001 | Loss: 3.4179 | LR: 1.73e-06 | 1330.8ms/step | 48.1 samples/s
1108
+ Step 9640/10001 | Loss: 3.4124 | LR: 1.64e-06 | 1333.3ms/step | 48.0 samples/s
1109
+ Step 9650/10001 | Loss: 3.4055 | LR: 1.55e-06 | 1326.5ms/step | 48.2 samples/s
1110
+ Step 9660/10001 | Loss: 3.3886 | LR: 1.46e-06 | 1325.5ms/step | 48.3 samples/s
1111
+ Step 9670/10001 | Loss: 3.3751 | LR: 1.38e-06 | 1333.0ms/step | 48.0 samples/s
1112
+ Step 9680/10001 | Loss: 3.3777 | LR: 1.30e-06 | 1336.5ms/step | 47.9 samples/s
1113
+ Step 9690/10001 | Loss: 3.4056 | LR: 1.22e-06 | 1331.4ms/step | 48.1 samples/s
1114
+ Step 9700/10001 | Loss: 3.4015 | LR: 1.14e-06 | 1329.5ms/step | 48.1 samples/s
1115
+ Step 9710/10001 | Loss: 3.4048 | LR: 1.06e-06 | 1330.5ms/step | 48.1 samples/s
1116
+ Step 9720/10001 | Loss: 3.4053 | LR: 9.93e-07 | 1331.8ms/step | 48.1 samples/s
1117
+ Step 9730/10001 | Loss: 3.3982 | LR: 9.24e-07 | 1331.2ms/step | 48.1 samples/s
1118
+ Step 9740/10001 | Loss: 3.3868 | LR: 8.57e-07 | 1329.1ms/step | 48.2 samples/s
1119
+ Step 9750/10001 | Loss: 3.4135 | LR: 7.92e-07 | 1330.5ms/step | 48.1 samples/s
1120
+ Validation at step 9750: Loss = 3.3994, Perplexity = 29.95
1121
+ Step 9760/10001 | Loss: 3.3962 | LR: 7.31e-07 | 26720.5ms/step | 2.4 samples/s
1122
+ Saved checkpoint to checkpoints/run_20260430_021312/checkpoint-9764
epfl_final/wandb/run-20260430_021315-cqkg8w61/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.1.post1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.4.3
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.58.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.3
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.3
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.49
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.0
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.12.2
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.7.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260430_021315-cqkg8w61/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-04-30T02:13:15.963087Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-6-0-1",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9371709865984",
19
+ "used": "4381362393088"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-98029aa4-c589-b65b-3ab7-6495256f31ed"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "w6c4b8mmqpwfn5x8nr7561y9vmdxch67"
36
+ }
epfl_final/wandb/run-20260430_021315-cqkg8w61/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"train/loss":3.3961692065000535,"_timestamp":1.7775384325587325e+09,"speed/samples_per_sec":2.3951657961936235,"train/global_step":9760,"eval/perplexity":29.94531025348752,"_runtime":23245.945873928,"train/epoch":0,"speed/steps_per_sec":0.03742446556552537,"speed/ms_per_step":26720.488452911377,"eval/loss":3.3993727264404296,"train/learning_rate":7.305906022035125e-07,"_step":9760,"_wandb":{"runtime":23245}}
epfl_final/wandb/run-20260430_022845-fnhn8tt1/files/output.log ADDED
@@ -0,0 +1,469 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
2
+ Initializing model from scratch (random weights)...
3
+ Checkpoint activations: False
4
+
5
+
6
+ ================================================================================
7
+ MODEL ARCHITECTURE
8
+ ================================================================================
9
+ Model config:
10
+ vocab_size: 151936
11
+ hidden_size: 1024
12
+ num_hidden_layers: 28
13
+ num_attention_heads: 16
14
+ num_key_value_heads: 8
15
+ head_dim: 128
16
+ intermediate_size: 3072
17
+ max_position_embeddings: 32768
18
+ rope_theta: 1000000
19
+ attention_type: sparse_stochastic
20
+ sparse_block_size: 128
21
+ sparse_num_random_blocks: 1
22
+ sparse_mask_pool_size: 1000
23
+ sparse_max_seq_length: 2048
24
+
25
+ Parameter count:
26
+ Total parameters: 596,049,920
27
+ Trainable parameters: 596,049,920
28
+ Non-trainable parameters: 0
29
+ Model size (BF16): 1.19 GB
30
+
31
+ --------------------------------------------------------------------------------
32
+ LAYER-BY-LAYER SUMMARY
33
+ --------------------------------------------------------------------------------
34
+ /mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
35
+
36
+ SOLUTION: Use torch.compile(flex_attention)(...)
37
+
38
+ If you want to debug your score_mod/mask_mod, you can set:
39
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
40
+
41
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
42
+ _warn_once(
43
+ ================================================================================================================================================================
44
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
45
+ ================================================================================================================================================================
46
+ Qwen3ForCausalLM [8, 2048] -- -- True
47
+ ├─Qwen3Model: 1-1 -- [8, 2048, 1024] -- True
48
+ │ └─Embedding: 2-1 [8, 2048] [8, 2048, 1024] 155,582,464 True
49
+ │ └─ModuleList: 2-2 -- -- -- True
50
+ │ │ └─Qwen3DecoderLayer: 3-1 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-2 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-3 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-4 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-5 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-6 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-7 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-8 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-9 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-10 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-11 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-12 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-13 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-14 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-15 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-16 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-17 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-18 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-19 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-20 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-21 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-22 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-23 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-24 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-25 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
75
+ │ │ └─Qwen3DecoderLayer: 3-26 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
76
+ │ │ └─Qwen3DecoderLayer: 3-27 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
77
+ │ │ └─Qwen3DecoderLayer: 3-28 [8, 2048, 1024] [8, 2048, 1024] 15,730,944 True
78
+ │ └─Qwen3RMSNorm: 2-3 [8, 2048, 1024] [8, 2048, 1024] 1,024 True
79
+ ├─Linear: 1-2 [8, 2048, 1024] [8, 2048, 151936] 155,582,464 True
80
+ ================================================================================================================================================================
81
+ Total params: 751,632,384
82
+ Trainable params: 751,632,384
83
+ Non-trainable params: 0
84
+ Total mult-adds (Units.GIGABYTES): 6.01
85
+ ================================================================================================================================================================
86
+ Input size (MB): 0.13
87
+ Forward/backward pass size (MB): 42035.31
88
+ Params size (MB): 1503.26
89
+ Estimated Total Size (MB): 43538.71
90
+ ================================================================================================================================================================
91
+ ================================================================================
92
+
93
+ Skipping flash-attention hook for sparse_stochastic mode
94
+ Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
95
+ Using pre-tokenized dataset (no runtime tokenization)
96
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
97
+ Loaded 781148 pre-tokenized examples
98
+ Dataset split: 624918 train, 156230 validation
99
+ Training examples: 624918
100
+ Validation examples: 156230
101
+ Steps per epoch: 9764
102
+ Total training steps: 10001
103
+ Starting training...
104
+ Step 10/10001 | Loss: 11.9292 | LR: 5.00e-05 | 6839.7ms/step | 9.4 samples/s
105
+ Step 20/10001 | Loss: 10.6763 | LR: 1.00e-04 | 1317.0ms/step | 48.6 samples/s
106
+ Step 30/10001 | Loss: 9.7177 | LR: 1.50e-04 | 1258.3ms/step | 50.9 samples/s
107
+ Step 40/10001 | Loss: 8.5572 | LR: 2.00e-04 | 1261.6ms/step | 50.7 samples/s
108
+ Step 50/10001 | Loss: 7.9713 | LR: 2.50e-04 | 1261.3ms/step | 50.7 samples/s
109
+ Step 60/10001 | Loss: 7.8931 | LR: 3.00e-04 | 1266.7ms/step | 50.5 samples/s
110
+ Step 70/10001 | Loss: 7.7058 | LR: 3.50e-04 | 1276.4ms/step | 50.1 samples/s
111
+ Step 80/10001 | Loss: 7.4909 | LR: 4.00e-04 | 1279.5ms/step | 50.0 samples/s
112
+ Step 90/10001 | Loss: 7.3501 | LR: 4.50e-04 | 1280.3ms/step | 50.0 samples/s
113
+ Step 100/10001 | Loss: 7.1459 | LR: 5.00e-04 | 1280.3ms/step | 50.0 samples/s
114
+ Step 110/10001 | Loss: 7.0341 | LR: 5.00e-04 | 1279.5ms/step | 50.0 samples/s
115
+ Step 120/10001 | Loss: 6.8862 | LR: 5.00e-04 | 1280.3ms/step | 50.0 samples/s
116
+ Step 130/10001 | Loss: 6.8153 | LR: 5.00e-04 | 1280.3ms/step | 50.0 samples/s
117
+ Step 140/10001 | Loss: 6.6850 | LR: 5.00e-04 | 1279.7ms/step | 50.0 samples/s
118
+ Step 150/10001 | Loss: 6.6209 | LR: 5.00e-04 | 1280.2ms/step | 50.0 samples/s
119
+ Step 160/10001 | Loss: 6.5967 | LR: 5.00e-04 | 1280.0ms/step | 50.0 samples/s
120
+ Step 170/10001 | Loss: 6.4750 | LR: 5.00e-04 | 1279.8ms/step | 50.0 samples/s
121
+ Step 180/10001 | Loss: 6.4544 | LR: 5.00e-04 | 1280.1ms/step | 50.0 samples/s
122
+ Step 190/10001 | Loss: 6.3411 | LR: 5.00e-04 | 1279.9ms/step | 50.0 samples/s
123
+ Step 200/10001 | Loss: 6.3324 | LR: 5.00e-04 | 1281.2ms/step | 50.0 samples/s
124
+ Step 210/10001 | Loss: 6.2524 | LR: 5.00e-04 | 1281.0ms/step | 50.0 samples/s
125
+ Step 220/10001 | Loss: 6.1806 | LR: 5.00e-04 | 1280.1ms/step | 50.0 samples/s
126
+ Step 230/10001 | Loss: 6.1774 | LR: 5.00e-04 | 1280.3ms/step | 50.0 samples/s
127
+ Step 240/10001 | Loss: 6.1034 | LR: 5.00e-04 | 1279.8ms/step | 50.0 samples/s
128
+ Step 250/10001 | Loss: 6.0245 | LR: 5.00e-04 | 1279.8ms/step | 50.0 samples/s
129
+ Validation at step 250: Loss = 6.0018, Perplexity = 404.18
130
+ Step 260/10001 | Loss: 5.9932 | LR: 5.00e-04 | 28358.4ms/step | 2.3 samples/s
131
+ Step 270/10001 | Loss: 5.9336 | LR: 5.00e-04 | 1279.9ms/step | 50.0 samples/s
132
+ Step 280/10001 | Loss: 5.8503 | LR: 5.00e-04 | 1279.7ms/step | 50.0 samples/s
133
+ Step 290/10001 | Loss: 5.8633 | LR: 5.00e-04 | 1284.2ms/step | 49.8 samples/s
134
+ Step 300/10001 | Loss: 5.8040 | LR: 4.99e-04 | 1280.1ms/step | 50.0 samples/s
135
+ Step 310/10001 | Loss: 5.7481 | LR: 4.99e-04 | 1280.0ms/step | 50.0 samples/s
136
+ Step 320/10001 | Loss: 5.6974 | LR: 4.99e-04 | 1280.1ms/step | 50.0 samples/s
137
+ Step 330/10001 | Loss: 5.7049 | LR: 4.99e-04 | 1281.9ms/step | 49.9 samples/s
138
+ Step 340/10001 | Loss: 5.6273 | LR: 4.99e-04 | 1279.7ms/step | 50.0 samples/s
139
+ Step 350/10001 | Loss: 5.6233 | LR: 4.99e-04 | 1280.2ms/step | 50.0 samples/s
140
+ Step 360/10001 | Loss: 5.5445 | LR: 4.99e-04 | 1282.2ms/step | 49.9 samples/s
141
+ Step 370/10001 | Loss: 5.5185 | LR: 4.99e-04 | 1280.1ms/step | 50.0 samples/s
142
+ Step 380/10001 | Loss: 5.4918 | LR: 4.99e-04 | 1279.8ms/step | 50.0 samples/s
143
+ Step 390/10001 | Loss: 5.4622 | LR: 4.99e-04 | 1281.9ms/step | 49.9 samples/s
144
+ Step 400/10001 | Loss: 5.4268 | LR: 4.99e-04 | 1280.1ms/step | 50.0 samples/s
145
+ Step 410/10001 | Loss: 5.3782 | LR: 4.99e-04 | 1279.8ms/step | 50.0 samples/s
146
+ Step 420/10001 | Loss: 5.3419 | LR: 4.99e-04 | 1280.4ms/step | 50.0 samples/s
147
+ Step 430/10001 | Loss: 5.3243 | LR: 4.99e-04 | 1279.7ms/step | 50.0 samples/s
148
+ Step 440/10001 | Loss: 5.2979 | LR: 4.99e-04 | 1280.1ms/step | 50.0 samples/s
149
+ Step 450/10001 | Loss: 5.2528 | LR: 4.98e-04 | 1280.0ms/step | 50.0 samples/s
150
+ Step 460/10001 | Loss: 5.2212 | LR: 4.98e-04 | 1279.8ms/step | 50.0 samples/s
151
+ Step 470/10001 | Loss: 5.2206 | LR: 4.98e-04 | 1280.0ms/step | 50.0 samples/s
152
+ Step 480/10001 | Loss: 5.1730 | LR: 4.98e-04 | 1280.3ms/step | 50.0 samples/s
153
+ Step 490/10001 | Loss: 5.1638 | LR: 4.98e-04 | 1280.3ms/step | 50.0 samples/s
154
+ Step 500/10001 | Loss: 5.1018 | LR: 4.98e-04 | 1279.6ms/step | 50.0 samples/s
155
+ Validation at step 500: Loss = 5.1055, Perplexity = 164.93
156
+ Step 510/10001 | Loss: 5.0917 | LR: 4.98e-04 | 26114.4ms/step | 2.5 samples/s
157
+ Step 520/10001 | Loss: 5.0722 | LR: 4.98e-04 | 1280.0ms/step | 50.0 samples/s
158
+ Step 530/10001 | Loss: 5.0497 | LR: 4.98e-04 | 1280.3ms/step | 50.0 samples/s
159
+ Step 540/10001 | Loss: 5.0130 | LR: 4.98e-04 | 1281.8ms/step | 49.9 samples/s
160
+ Step 550/10001 | Loss: 4.9890 | LR: 4.97e-04 | 1279.9ms/step | 50.0 samples/s
161
+ Step 560/10001 | Loss: 4.9947 | LR: 4.97e-04 | 1280.3ms/step | 50.0 samples/s
162
+ Step 570/10001 | Loss: 4.9507 | LR: 4.97e-04 | 1281.7ms/step | 49.9 samples/s
163
+ Step 580/10001 | Loss: 4.9487 | LR: 4.97e-04 | 1280.1ms/step | 50.0 samples/s
164
+ Step 590/10001 | Loss: 4.9233 | LR: 4.97e-04 | 1279.8ms/step | 50.0 samples/s
165
+ Step 600/10001 | Loss: 4.9140 | LR: 4.97e-04 | 1280.5ms/step | 50.0 samples/s
166
+ Step 610/10001 | Loss: 4.8901 | LR: 4.97e-04 | 1279.7ms/step | 50.0 samples/s
167
+ Step 620/10001 | Loss: 4.8539 | LR: 4.97e-04 | 1279.9ms/step | 50.0 samples/s
168
+ Step 630/10001 | Loss: 4.8393 | LR: 4.96e-04 | 1280.3ms/step | 50.0 samples/s
169
+ Step 640/10001 | Loss: 4.8095 | LR: 4.96e-04 | 1279.7ms/step | 50.0 samples/s
170
+ Step 650/10001 | Loss: 4.7907 | LR: 4.96e-04 | 1280.2ms/step | 50.0 samples/s
171
+ Step 660/10001 | Loss: 4.8075 | LR: 4.96e-04 | 1281.4ms/step | 49.9 samples/s
172
+ Step 670/10001 | Loss: 4.7642 | LR: 4.96e-04 | 1280.5ms/step | 50.0 samples/s
173
+ Step 680/10001 | Loss: 4.7573 | LR: 4.96e-04 | 1280.2ms/step | 50.0 samples/s
174
+ Step 690/10001 | Loss: 4.7433 | LR: 4.96e-04 | 1280.0ms/step | 50.0 samples/s
175
+ Step 700/10001 | Loss: 4.7582 | LR: 4.95e-04 | 1279.8ms/step | 50.0 samples/s
176
+ Step 710/10001 | Loss: 4.7240 | LR: 4.95e-04 | 1280.0ms/step | 50.0 samples/s
177
+ Step 720/10001 | Loss: 4.7314 | LR: 4.95e-04 | 1280.1ms/step | 50.0 samples/s
178
+ Step 730/10001 | Loss: 4.7143 | LR: 4.95e-04 | 1280.2ms/step | 50.0 samples/s
179
+ Step 740/10001 | Loss: 4.6933 | LR: 4.95e-04 | 1279.9ms/step | 50.0 samples/s
180
+ Step 750/10001 | Loss: 4.6848 | LR: 4.95e-04 | 1279.8ms/step | 50.0 samples/s
181
+ Validation at step 750: Loss = 4.6640, Perplexity = 106.06
182
+ Step 760/10001 | Loss: 4.6333 | LR: 4.95e-04 | 26224.3ms/step | 2.4 samples/s
183
+ Step 770/10001 | Loss: 4.6685 | LR: 4.94e-04 | 1290.0ms/step | 49.6 samples/s
184
+ Step 780/10001 | Loss: 4.6211 | LR: 4.94e-04 | 1280.4ms/step | 50.0 samples/s
185
+ Step 790/10001 | Loss: 4.6212 | LR: 4.94e-04 | 1280.0ms/step | 50.0 samples/s
186
+ Step 800/10001 | Loss: 4.6252 | LR: 4.94e-04 | 1286.4ms/step | 49.7 samples/s
187
+ Step 810/10001 | Loss: 4.6061 | LR: 4.94e-04 | 1285.4ms/step | 49.8 samples/s
188
+ Step 820/10001 | Loss: 4.5710 | LR: 4.94e-04 | 1280.0ms/step | 50.0 samples/s
189
+ Step 830/10001 | Loss: 4.5870 | LR: 4.93e-04 | 1280.6ms/step | 50.0 samples/s
190
+ Step 840/10001 | Loss: 4.5610 | LR: 4.93e-04 | 1287.3ms/step | 49.7 samples/s
191
+ Step 850/10001 | Loss: 4.5716 | LR: 4.93e-04 | 1280.3ms/step | 50.0 samples/s
192
+ Step 860/10001 | Loss: 4.5353 | LR: 4.93e-04 | 1279.9ms/step | 50.0 samples/s
193
+ Step 870/10001 | Loss: 4.5158 | LR: 4.93e-04 | 1288.1ms/step | 49.7 samples/s
194
+ Step 880/10001 | Loss: 4.5326 | LR: 4.92e-04 | 1279.8ms/step | 50.0 samples/s
195
+ Step 890/10001 | Loss: 4.5451 | LR: 4.92e-04 | 1279.9ms/step | 50.0 samples/s
196
+ Step 900/10001 | Loss: 4.4985 | LR: 4.92e-04 | 1282.2ms/step | 49.9 samples/s
197
+ Step 910/10001 | Loss: 4.5150 | LR: 4.92e-04 | 1283.8ms/step | 49.9 samples/s
198
+ Step 920/10001 | Loss: 4.4696 | LR: 4.92e-04 | 1280.1ms/step | 50.0 samples/s
199
+ Step 930/10001 | Loss: 4.4776 | LR: 4.91e-04 | 1280.9ms/step | 50.0 samples/s
200
+ Step 940/10001 | Loss: 4.4821 | LR: 4.91e-04 | 1289.0ms/step | 49.7 samples/s
201
+ Step 950/10001 | Loss: 4.4605 | LR: 4.91e-04 | 1280.1ms/step | 50.0 samples/s
202
+ Step 960/10001 | Loss: 4.4239 | LR: 4.91e-04 | 1280.0ms/step | 50.0 samples/s
203
+ Step 970/10001 | Loss: 4.4375 | LR: 4.91e-04 | 1288.1ms/step | 49.7 samples/s
204
+ Step 980/10001 | Loss: 4.4072 | LR: 4.90e-04 | 1279.8ms/step | 50.0 samples/s
205
+ Step 990/10001 | Loss: 4.4406 | LR: 4.90e-04 | 1280.2ms/step | 50.0 samples/s
206
+ Step 1000/10001 | Loss: 4.4350 | LR: 4.90e-04 | 1284.2ms/step | 49.8 samples/s
207
+ Validation at step 1000: Loss = 4.4255, Perplexity = 83.56
208
+ Step 1010/10001 | Loss: 4.4021 | LR: 4.90e-04 | 26220.6ms/step | 2.4 samples/s
209
+ Step 1020/10001 | Loss: 4.4169 | LR: 4.89e-04 | 1283.2ms/step | 49.9 samples/s
210
+ Step 1030/10001 | Loss: 4.3913 | LR: 4.89e-04 | 1280.0ms/step | 50.0 samples/s
211
+ Step 1040/10001 | Loss: 4.3760 | LR: 4.89e-04 | 1286.5ms/step | 49.7 samples/s
212
+ Step 1050/10001 | Loss: 4.3897 | LR: 4.89e-04 | 1287.8ms/step | 49.7 samples/s
213
+ Step 1060/10001 | Loss: 4.3926 | LR: 4.88e-04 | 1279.9ms/step | 50.0 samples/s
214
+ Step 1070/10001 | Loss: 4.4011 | LR: 4.88e-04 | 1279.9ms/step | 50.0 samples/s
215
+ Step 1080/10001 | Loss: 4.3546 | LR: 4.88e-04 | 1287.8ms/step | 49.7 samples/s
216
+ Step 1090/10001 | Loss: 4.3553 | LR: 4.88e-04 | 1280.1ms/step | 50.0 samples/s
217
+ Step 1100/10001 | Loss: 4.3440 | LR: 4.88e-04 | 1284.2ms/step | 49.8 samples/s
218
+ Step 1110/10001 | Loss: 4.3608 | LR: 4.87e-04 | 1288.2ms/step | 49.7 samples/s
219
+ Step 1120/10001 | Loss: 4.3270 | LR: 4.87e-04 | 1287.8ms/step | 49.7 samples/s
220
+ Step 1130/10001 | Loss: 4.3481 | LR: 4.87e-04 | 1287.8ms/step | 49.7 samples/s
221
+ Step 1140/10001 | Loss: 4.3453 | LR: 4.87e-04 | 1285.3ms/step | 49.8 samples/s
222
+ Step 1150/10001 | Loss: 4.3225 | LR: 4.86e-04 | 1290.7ms/step | 49.6 samples/s
223
+ Step 1160/10001 | Loss: 4.3194 | LR: 4.86e-04 | 1280.5ms/step | 50.0 samples/s
224
+ Step 1170/10001 | Loss: 4.2929 | LR: 4.86e-04 | 1279.7ms/step | 50.0 samples/s
225
+ Step 1180/10001 | Loss: 4.2953 | LR: 4.85e-04 | 1284.8ms/step | 49.8 samples/s
226
+ Step 1190/10001 | Loss: 4.3131 | LR: 4.85e-04 | 1281.2ms/step | 50.0 samples/s
227
+ Step 1200/10001 | Loss: 4.2884 | LR: 4.85e-04 | 1280.1ms/step | 50.0 samples/s
228
+ Step 1210/10001 | Loss: 4.3225 | LR: 4.85e-04 | 1282.6ms/step | 49.9 samples/s
229
+ Step 1220/10001 | Loss: 4.3075 | LR: 4.84e-04 | 1283.3ms/step | 49.9 samples/s
230
+ Step 1230/10001 | Loss: 4.3041 | LR: 4.84e-04 | 1280.4ms/step | 50.0 samples/s
231
+ Step 1240/10001 | Loss: 4.2901 | LR: 4.84e-04 | 1280.0ms/step | 50.0 samples/s
232
+ Step 1250/10001 | Loss: 4.2696 | LR: 4.84e-04 | 1286.0ms/step | 49.8 samples/s
233
+ Validation at step 1250: Loss = 4.2602, Perplexity = 70.82
234
+ Step 1260/10001 | Loss: 4.2933 | LR: 4.83e-04 | 26242.3ms/step | 2.4 samples/s
235
+ Step 1270/10001 | Loss: 4.2459 | LR: 4.83e-04 | 1280.1ms/step | 50.0 samples/s
236
+ Step 1280/10001 | Loss: 4.2402 | LR: 4.83e-04 | 1279.8ms/step | 50.0 samples/s
237
+ Step 1290/10001 | Loss: 4.2521 | LR: 4.82e-04 | 1289.8ms/step | 49.6 samples/s
238
+ Step 1300/10001 | Loss: 4.2478 | LR: 4.82e-04 | 1280.2ms/step | 50.0 samples/s
239
+ Step 1310/10001 | Loss: 4.2419 | LR: 4.82e-04 | 1280.0ms/step | 50.0 samples/s
240
+ Step 1320/10001 | Loss: 4.2485 | LR: 4.82e-04 | 1286.0ms/step | 49.8 samples/s
241
+ Step 1330/10001 | Loss: 4.2420 | LR: 4.81e-04 | 1283.9ms/step | 49.8 samples/s
242
+ Step 1340/10001 | Loss: 4.2477 | LR: 4.81e-04 | 1280.1ms/step | 50.0 samples/s
243
+ Step 1350/10001 | Loss: 4.2135 | LR: 4.81e-04 | 1282.2ms/step | 49.9 samples/s
244
+ Step 1360/10001 | Loss: 4.2085 | LR: 4.80e-04 | 1285.8ms/step | 49.8 samples/s
245
+ Step 1370/10001 | Loss: 4.1930 | LR: 4.80e-04 | 1279.8ms/step | 50.0 samples/s
246
+ Step 1380/10001 | Loss: 4.1861 | LR: 4.80e-04 | 1280.2ms/step | 50.0 samples/s
247
+ Step 1390/10001 | Loss: 4.1841 | LR: 4.79e-04 | 1291.6ms/step | 49.6 samples/s
248
+ Step 1400/10001 | Loss: 4.1521 | LR: 4.79e-04 | 1280.6ms/step | 50.0 samples/s
249
+ Step 1410/10001 | Loss: 4.1971 | LR: 4.79e-04 | 1280.1ms/step | 50.0 samples/s
250
+ Step 1420/10001 | Loss: 4.1773 | LR: 4.78e-04 | 1288.1ms/step | 49.7 samples/s
251
+ Step 1430/10001 | Loss: 4.1841 | LR: 4.78e-04 | 1285.8ms/step | 49.8 samples/s
252
+ Step 1440/10001 | Loss: 4.1703 | LR: 4.78e-04 | 1279.9ms/step | 50.0 samples/s
253
+ Step 1450/10001 | Loss: 4.1715 | LR: 4.77e-04 | 1281.6ms/step | 49.9 samples/s
254
+ Step 1460/10001 | Loss: 4.1894 | LR: 4.77e-04 | 1290.4ms/step | 49.6 samples/s
255
+ Step 1470/10001 | Loss: 4.1708 | LR: 4.77e-04 | 1280.0ms/step | 50.0 samples/s
256
+ Step 1480/10001 | Loss: 4.1251 | LR: 4.76e-04 | 1280.0ms/step | 50.0 samples/s
257
+ Step 1490/10001 | Loss: 4.1570 | LR: 4.76e-04 | 1289.4ms/step | 49.6 samples/s
258
+ Step 1500/10001 | Loss: 4.1439 | LR: 4.76e-04 | 1280.5ms/step | 50.0 samples/s
259
+ Validation at step 1500: Loss = 4.1477, Perplexity = 63.29
260
+ Step 1510/10001 | Loss: 4.1576 | LR: 4.75e-04 | 26244.3ms/step | 2.4 samples/s
261
+ Step 1520/10001 | Loss: 4.1287 | LR: 4.75e-04 | 1280.1ms/step | 50.0 samples/s
262
+ Step 1530/10001 | Loss: 4.1334 | LR: 4.75e-04 | 1287.2ms/step | 49.7 samples/s
263
+ Step 1540/10001 | Loss: 4.1219 | LR: 4.74e-04 | 1284.6ms/step | 49.8 samples/s
264
+ Step 1550/10001 | Loss: 4.1073 | LR: 4.74e-04 | 1280.2ms/step | 50.0 samples/s
265
+ Step 1560/10001 | Loss: 4.1301 | LR: 4.74e-04 | 1282.1ms/step | 49.9 samples/s
266
+ Step 1570/10001 | Loss: 4.1142 | LR: 4.73e-04 | 1285.9ms/step | 49.8 samples/s
267
+ Step 1580/10001 | Loss: 4.1019 | LR: 4.73e-04 | 1280.0ms/step | 50.0 samples/s
268
+ Step 1590/10001 | Loss: 4.1177 | LR: 4.73e-04 | 1280.0ms/step | 50.0 samples/s
269
+ Step 1600/10001 | Loss: 4.0984 | LR: 4.72e-04 | 1285.8ms/step | 49.8 samples/s
270
+ Step 1610/10001 | Loss: 4.0879 | LR: 4.72e-04 | 1280.1ms/step | 50.0 samples/s
271
+ Step 1620/10001 | Loss: 4.1031 | LR: 4.71e-04 | 1280.0ms/step | 50.0 samples/s
272
+ Step 1630/10001 | Loss: 4.0913 | LR: 4.71e-04 | 1282.2ms/step | 49.9 samples/s
273
+ Step 1640/10001 | Loss: 4.0812 | LR: 4.71e-04 | 1282.3ms/step | 49.9 samples/s
274
+ Step 1650/10001 | Loss: 4.0821 | LR: 4.70e-04 | 1281.8ms/step | 49.9 samples/s
275
+ Step 1660/10001 | Loss: 4.0961 | LR: 4.70e-04 | 1281.9ms/step | 49.9 samples/s
276
+ Step 1670/10001 | Loss: 4.0564 | LR: 4.70e-04 | 1288.0ms/step | 49.7 samples/s
277
+ Step 1680/10001 | Loss: 4.0834 | LR: 4.69e-04 | 1279.9ms/step | 50.0 samples/s
278
+ Step 1690/10001 | Loss: 4.0677 | LR: 4.69e-04 | 1279.9ms/step | 50.0 samples/s
279
+ Step 1700/10001 | Loss: 4.0670 | LR: 4.68e-04 | 1287.7ms/step | 49.7 samples/s
280
+ Step 1710/10001 | Loss: 4.0578 | LR: 4.68e-04 | 1280.5ms/step | 50.0 samples/s
281
+ Step 1720/10001 | Loss: 4.0568 | LR: 4.68e-04 | 1280.0ms/step | 50.0 samples/s
282
+ Step 1730/10001 | Loss: 4.0844 | LR: 4.67e-04 | 1284.3ms/step | 49.8 samples/s
283
+ Step 1740/10001 | Loss: 4.0576 | LR: 4.67e-04 | 1283.6ms/step | 49.9 samples/s
284
+ Step 1750/10001 | Loss: 4.0823 | LR: 4.67e-04 | 1280.0ms/step | 50.0 samples/s
285
+ Validation at step 1750: Loss = 4.0606, Perplexity = 58.01
286
+ Step 1760/10001 | Loss: 4.0769 | LR: 4.66e-04 | 26252.2ms/step | 2.4 samples/s
287
+ Step 1770/10001 | Loss: 4.0315 | LR: 4.66e-04 | 1282.1ms/step | 49.9 samples/s
288
+ Step 1780/10001 | Loss: 4.0484 | LR: 4.65e-04 | 1285.9ms/step | 49.8 samples/s
289
+ Step 1790/10001 | Loss: 4.0540 | LR: 4.65e-04 | 1279.8ms/step | 50.0 samples/s
290
+ Step 1800/10001 | Loss: 4.0515 | LR: 4.65e-04 | 1280.3ms/step | 50.0 samples/s
291
+ Step 1810/10001 | Loss: 4.0291 | LR: 4.64e-04 | 1297.6ms/step | 49.3 samples/s
292
+ Step 1820/10001 | Loss: 4.0248 | LR: 4.64e-04 | 1280.3ms/step | 50.0 samples/s
293
+ Step 1830/10001 | Loss: 4.0469 | LR: 4.63e-04 | 1279.9ms/step | 50.0 samples/s
294
+ Step 1840/10001 | Loss: 4.0173 | LR: 4.63e-04 | 1286.4ms/step | 49.7 samples/s
295
+ Step 1850/10001 | Loss: 4.0236 | LR: 4.62e-04 | 1283.6ms/step | 49.9 samples/s
296
+ Step 1860/10001 | Loss: 4.0235 | LR: 4.62e-04 | 1280.0ms/step | 50.0 samples/s
297
+ Step 1870/10001 | Loss: 4.0277 | LR: 4.62e-04 | 1284.4ms/step | 49.8 samples/s
298
+ Step 1880/10001 | Loss: 4.0212 | LR: 4.61e-04 | 1287.7ms/step | 49.7 samples/s
299
+ Step 1890/10001 | Loss: 4.0152 | LR: 4.61e-04 | 1279.8ms/step | 50.0 samples/s
300
+ Step 1900/10001 | Loss: 4.0342 | LR: 4.60e-04 | 1282.4ms/step | 49.9 samples/s
301
+ Step 1910/10001 | Loss: 4.0243 | LR: 4.60e-04 | 1289.8ms/step | 49.6 samples/s
302
+ Step 1920/10001 | Loss: 4.0289 | LR: 4.59e-04 | 1279.7ms/step | 50.0 samples/s
303
+ Step 1930/10001 | Loss: 4.0215 | LR: 4.59e-04 | 1280.3ms/step | 50.0 samples/s
304
+ Step 1940/10001 | Loss: 3.9909 | LR: 4.59e-04 | 1284.2ms/step | 49.8 samples/s
305
+ Step 1950/10001 | Loss: 4.0039 | LR: 4.58e-04 | 1287.5ms/step | 49.7 samples/s
306
+ Step 1960/10001 | Loss: 4.0098 | LR: 4.58e-04 | 1280.3ms/step | 50.0 samples/s
307
+ Step 1970/10001 | Loss: 3.9901 | LR: 4.57e-04 | 1281.8ms/step | 49.9 samples/s
308
+ Step 1980/10001 | Loss: 4.0105 | LR: 4.57e-04 | 1288.1ms/step | 49.7 samples/s
309
+ Step 1990/10001 | Loss: 3.9942 | LR: 4.56e-04 | 1281.9ms/step | 49.9 samples/s
310
+ Step 2000/10001 | Loss: 3.9875 | LR: 4.56e-04 | 1280.0ms/step | 50.0 samples/s
311
+ Validation at step 2000: Loss = 3.9908, Perplexity = 54.10
312
+ Step 2010/10001 | Loss: 3.9873 | LR: 4.55e-04 | 26228.2ms/step | 2.4 samples/s
313
+ Step 2020/10001 | Loss: 4.0149 | LR: 4.55e-04 | 1286.8ms/step | 49.7 samples/s
314
+ Step 2030/10001 | Loss: 3.9765 | LR: 4.55e-04 | 1279.3ms/step | 50.0 samples/s
315
+ Step 2040/10001 | Loss: 3.9607 | LR: 4.54e-04 | 1279.9ms/step | 50.0 samples/s
316
+ Step 2050/10001 | Loss: 3.9911 | LR: 4.54e-04 | 1282.6ms/step | 49.9 samples/s
317
+ Step 2060/10001 | Loss: 3.9811 | LR: 4.53e-04 | 1283.4ms/step | 49.9 samples/s
318
+ Step 2070/10001 | Loss: 3.9612 | LR: 4.53e-04 | 1280.2ms/step | 50.0 samples/s
319
+ Step 2080/10001 | Loss: 3.9904 | LR: 4.52e-04 | 1280.1ms/step | 50.0 samples/s
320
+ Step 2090/10001 | Loss: 3.9591 | LR: 4.52e-04 | 1283.6ms/step | 49.9 samples/s
321
+ Step 2100/10001 | Loss: 3.9630 | LR: 4.51e-04 | 1280.1ms/step | 50.0 samples/s
322
+ Step 2110/10001 | Loss: 3.9463 | LR: 4.51e-04 | 1279.9ms/step | 50.0 samples/s
323
+ Step 2120/10001 | Loss: 3.9608 | LR: 4.50e-04 | 1284.2ms/step | 49.8 samples/s
324
+ Step 2130/10001 | Loss: 3.9412 | LR: 4.50e-04 | 1280.0ms/step | 50.0 samples/s
325
+ Step 2140/10001 | Loss: 3.9632 | LR: 4.49e-04 | 1280.1ms/step | 50.0 samples/s
326
+ Step 2150/10001 | Loss: 3.9455 | LR: 4.49e-04 | 1281.0ms/step | 50.0 samples/s
327
+ Step 2160/10001 | Loss: 3.9443 | LR: 4.48e-04 | 1283.0ms/step | 49.9 samples/s
328
+ Step 2170/10001 | Loss: 3.9639 | LR: 4.48e-04 | 1279.9ms/step | 50.0 samples/s
329
+ Step 2180/10001 | Loss: 3.9620 | LR: 4.48e-04 | 1280.4ms/step | 50.0 samples/s
330
+ Step 2190/10001 | Loss: 3.9478 | LR: 4.47e-04 | 1283.6ms/step | 49.9 samples/s
331
+ Step 2200/10001 | Loss: 3.9560 | LR: 4.47e-04 | 1280.1ms/step | 50.0 samples/s
332
+ Step 2210/10001 | Loss: 3.9129 | LR: 4.46e-04 | 1280.0ms/step | 50.0 samples/s
333
+ Step 2220/10001 | Loss: 3.9506 | LR: 4.46e-04 | 1286.6ms/step | 49.7 samples/s
334
+ Step 2230/10001 | Loss: 3.9555 | LR: 4.45e-04 | 1281.5ms/step | 49.9 samples/s
335
+ Step 2240/10001 | Loss: 3.9299 | LR: 4.45e-04 | 1279.9ms/step | 50.0 samples/s
336
+ Step 2250/10001 | Loss: 3.8973 | LR: 4.44e-04 | 1282.8ms/step | 49.9 samples/s
337
+ Validation at step 2250: Loss = 3.9337, Perplexity = 51.09
338
+ Step 2260/10001 | Loss: 3.9349 | LR: 4.44e-04 | 26197.2ms/step | 2.4 samples/s
339
+ Step 2270/10001 | Loss: 3.9495 | LR: 4.43e-04 | 1280.2ms/step | 50.0 samples/s
340
+ Step 2280/10001 | Loss: 3.9206 | LR: 4.43e-04 | 1279.7ms/step | 50.0 samples/s
341
+ Step 2290/10001 | Loss: 3.9461 | LR: 4.42e-04 | 1280.0ms/step | 50.0 samples/s
342
+ Step 2300/10001 | Loss: 3.9585 | LR: 4.42e-04 | 1284.3ms/step | 49.8 samples/s
343
+ Step 2310/10001 | Loss: 3.9466 | LR: 4.41e-04 | 1281.8ms/step | 49.9 samples/s
344
+ Step 2320/10001 | Loss: 3.9059 | LR: 4.40e-04 | 1279.9ms/step | 50.0 samples/s
345
+ Step 2330/10001 | Loss: 3.9075 | LR: 4.40e-04 | 1280.5ms/step | 50.0 samples/s
346
+ Step 2340/10001 | Loss: 3.9198 | LR: 4.39e-04 | 1285.4ms/step | 49.8 samples/s
347
+ Step 2350/10001 | Loss: 3.9359 | LR: 4.39e-04 | 1280.3ms/step | 50.0 samples/s
348
+ Step 2360/10001 | Loss: 3.9224 | LR: 4.38e-04 | 1279.7ms/step | 50.0 samples/s
349
+ Step 2370/10001 | Loss: 3.9284 | LR: 4.38e-04 | 1280.7ms/step | 50.0 samples/s
350
+ Step 2380/10001 | Loss: 3.9040 | LR: 4.37e-04 | 1279.4ms/step | 50.0 samples/s
351
+ Step 2390/10001 | Loss: 3.8870 | LR: 4.37e-04 | 1280.0ms/step | 50.0 samples/s
352
+ Step 2400/10001 | Loss: 3.8941 | LR: 4.36e-04 | 1280.0ms/step | 50.0 samples/s
353
+ Step 2410/10001 | Loss: 3.9208 | LR: 4.36e-04 | 1282.2ms/step | 49.9 samples/s
354
+ Step 2420/10001 | Loss: 3.9009 | LR: 4.35e-04 | 1279.8ms/step | 50.0 samples/s
355
+ Step 2430/10001 | Loss: 3.8909 | LR: 4.35e-04 | 1280.0ms/step | 50.0 samples/s
356
+ Step 2440/10001 | Loss: 3.8819 | LR: 4.34e-04 | 1282.4ms/step | 49.9 samples/s
357
+ Step 2450/10001 | Loss: 3.8988 | LR: 4.34e-04 | 1279.8ms/step | 50.0 samples/s
358
+ Step 2460/10001 | Loss: 3.8847 | LR: 4.33e-04 | 1280.1ms/step | 50.0 samples/s
359
+ Step 2470/10001 | Loss: 3.9057 | LR: 4.33e-04 | 1280.4ms/step | 50.0 samples/s
360
+ Step 2480/10001 | Loss: 3.9050 | LR: 4.32e-04 | 1279.5ms/step | 50.0 samples/s
361
+ Step 2490/10001 | Loss: 3.8921 | LR: 4.31e-04 | 1280.2ms/step | 50.0 samples/s
362
+ Step 2500/10001 | Loss: 3.8899 | LR: 4.31e-04 | 1281.1ms/step | 50.0 samples/s
363
+ Validation at step 2500: Loss = 3.8864, Perplexity = 48.74
364
+ Saved checkpoint to checkpoints/run_20260430_022841/checkpoint-2500
365
+ Step 2510/10001 | Loss: 3.8721 | LR: 4.30e-04 | 26723.0ms/step | 2.4 samples/s
366
+ Step 2520/10001 | Loss: 3.8600 | LR: 4.30e-04 | 1280.3ms/step | 50.0 samples/s
367
+ Step 2530/10001 | Loss: 3.8799 | LR: 4.29e-04 | 1281.6ms/step | 49.9 samples/s
368
+ Step 2540/10001 | Loss: 3.8825 | LR: 4.29e-04 | 1279.9ms/step | 50.0 samples/s
369
+ Step 2550/10001 | Loss: 3.8963 | LR: 4.28e-04 | 1280.3ms/step | 50.0 samples/s
370
+ Step 2560/10001 | Loss: 3.8390 | LR: 4.28e-04 | 1283.6ms/step | 49.9 samples/s
371
+ Step 2570/10001 | Loss: 3.8741 | LR: 4.27e-04 | 1280.2ms/step | 50.0 samples/s
372
+ Step 2580/10001 | Loss: 3.8608 | LR: 4.27e-04 | 1280.2ms/step | 50.0 samples/s
373
+ Step 2590/10001 | Loss: 3.8709 | LR: 4.26e-04 | 1279.9ms/step | 50.0 samples/s
374
+ Step 2600/10001 | Loss: 3.8542 | LR: 4.25e-04 | 1279.9ms/step | 50.0 samples/s
375
+ Step 2610/10001 | Loss: 3.8931 | LR: 4.25e-04 | 1280.1ms/step | 50.0 samples/s
376
+ Step 2620/10001 | Loss: 3.8539 | LR: 4.24e-04 | 1283.2ms/step | 49.9 samples/s
377
+ Step 2630/10001 | Loss: 3.8451 | LR: 4.24e-04 | 1280.8ms/step | 50.0 samples/s
378
+ Step 2640/10001 | Loss: 3.8524 | LR: 4.23e-04 | 1280.1ms/step | 50.0 samples/s
379
+ Step 2650/10001 | Loss: 3.8349 | LR: 4.23e-04 | 1282.4ms/step | 49.9 samples/s
380
+ Step 2660/10001 | Loss: 3.8620 | LR: 4.22e-04 | 1281.6ms/step | 49.9 samples/s
381
+ Step 2670/10001 | Loss: 3.8890 | LR: 4.21e-04 | 1280.1ms/step | 50.0 samples/s
382
+ Step 2680/10001 | Loss: 3.8507 | LR: 4.21e-04 | 1279.9ms/step | 50.0 samples/s
383
+ Step 2690/10001 | Loss: 3.8348 | LR: 4.20e-04 | 1282.5ms/step | 49.9 samples/s
384
+ Step 2700/10001 | Loss: 3.8478 | LR: 4.20e-04 | 1279.5ms/step | 50.0 samples/s
385
+ Step 2710/10001 | Loss: 3.8431 | LR: 4.19e-04 | 1279.9ms/step | 50.0 samples/s
386
+ Step 2720/10001 | Loss: 3.8270 | LR: 4.18e-04 | 1280.4ms/step | 50.0 samples/s
387
+ Step 2730/10001 | Loss: 3.8341 | LR: 4.18e-04 | 1279.7ms/step | 50.0 samples/s
388
+ Step 2740/10001 | Loss: 3.8476 | LR: 4.17e-04 | 1279.9ms/step | 50.0 samples/s
389
+ Step 2750/10001 | Loss: 3.8574 | LR: 4.17e-04 | 1280.2ms/step | 50.0 samples/s
390
+ Validation at step 2750: Loss = 3.8438, Perplexity = 46.70
391
+ Step 2760/10001 | Loss: 3.8535 | LR: 4.16e-04 | 26170.4ms/step | 2.4 samples/s
392
+ Step 2770/10001 | Loss: 3.8558 | LR: 4.16e-04 | 1283.7ms/step | 49.9 samples/s
393
+ Step 2780/10001 | Loss: 3.8326 | LR: 4.15e-04 | 1279.7ms/step | 50.0 samples/s
394
+ Step 2790/10001 | Loss: 3.8644 | LR: 4.14e-04 | 1280.0ms/step | 50.0 samples/s
395
+ Step 2800/10001 | Loss: 3.8366 | LR: 4.14e-04 | 1280.3ms/step | 50.0 samples/s
396
+ Step 2810/10001 | Loss: 3.8204 | LR: 4.13e-04 | 1279.9ms/step | 50.0 samples/s
397
+ Step 2820/10001 | Loss: 3.8425 | LR: 4.13e-04 | 1280.1ms/step | 50.0 samples/s
398
+ Step 2830/10001 | Loss: 3.8443 | LR: 4.12e-04 | 1282.2ms/step | 49.9 samples/s
399
+ Step 2840/10001 | Loss: 3.8377 | LR: 4.11e-04 | 1279.9ms/step | 50.0 samples/s
400
+ Step 2850/10001 | Loss: 3.8281 | LR: 4.11e-04 | 1279.8ms/step | 50.0 samples/s
401
+ Step 2860/10001 | Loss: 3.8345 | LR: 4.10e-04 | 1280.5ms/step | 50.0 samples/s
402
+ Step 2870/10001 | Loss: 3.8480 | LR: 4.09e-04 | 1281.7ms/step | 49.9 samples/s
403
+ Step 2880/10001 | Loss: 3.8542 | LR: 4.09e-04 | 1280.2ms/step | 50.0 samples/s
404
+ Step 2890/10001 | Loss: 3.8467 | LR: 4.08e-04 | 1279.7ms/step | 50.0 samples/s
405
+ Step 2900/10001 | Loss: 3.8080 | LR: 4.08e-04 | 1282.2ms/step | 49.9 samples/s
406
+ Step 2910/10001 | Loss: 3.8096 | LR: 4.07e-04 | 1280.0ms/step | 50.0 samples/s
407
+ Step 2920/10001 | Loss: 3.8222 | LR: 4.06e-04 | 1280.1ms/step | 50.0 samples/s
408
+ Step 2930/10001 | Loss: 3.8296 | LR: 4.06e-04 | 1280.4ms/step | 50.0 samples/s
409
+ Step 2940/10001 | Loss: 3.7873 | LR: 4.05e-04 | 1281.5ms/step | 49.9 samples/s
410
+ Step 2950/10001 | Loss: 3.7950 | LR: 4.05e-04 | 1279.9ms/step | 50.0 samples/s
411
+ Step 2960/10001 | Loss: 3.8070 | LR: 4.04e-04 | 1280.2ms/step | 50.0 samples/s
412
+ Step 2970/10001 | Loss: 3.8138 | LR: 4.03e-04 | 1283.8ms/step | 49.9 samples/s
413
+ Step 2980/10001 | Loss: 3.8103 | LR: 4.03e-04 | 1280.1ms/step | 50.0 samples/s
414
+ Step 2990/10001 | Loss: 3.8178 | LR: 4.02e-04 | 1279.8ms/step | 50.0 samples/s
415
+ Step 3000/10001 | Loss: 3.7963 | LR: 4.01e-04 | 1286.1ms/step | 49.8 samples/s
416
+ Validation at step 3000: Loss = 3.8069, Perplexity = 45.01
417
+ Step 3010/10001 | Loss: 3.8348 | LR: 4.01e-04 | 26158.1ms/step | 2.4 samples/s
418
+ Step 3020/10001 | Loss: 3.8156 | LR: 4.00e-04 | 1280.1ms/step | 50.0 samples/s
419
+ Step 3030/10001 | Loss: 3.8000 | LR: 4.00e-04 | 1279.9ms/step | 50.0 samples/s
420
+ Step 3040/10001 | Loss: 3.7801 | LR: 3.99e-04 | 1285.8ms/step | 49.8 samples/s
421
+ Step 3050/10001 | Loss: 3.8028 | LR: 3.98e-04 | 1280.3ms/step | 50.0 samples/s
422
+ Step 3060/10001 | Loss: 3.8148 | LR: 3.98e-04 | 1279.7ms/step | 50.0 samples/s
423
+ Step 3070/10001 | Loss: 3.7660 | LR: 3.97e-04 | 1282.8ms/step | 49.9 samples/s
424
+ Step 3080/10001 | Loss: 3.7813 | LR: 3.96e-04 | 1281.5ms/step | 49.9 samples/s
425
+ Step 3090/10001 | Loss: 3.7986 | LR: 3.96e-04 | 1279.9ms/step | 50.0 samples/s
426
+ Step 3100/10001 | Loss: 3.7763 | LR: 3.95e-04 | 1280.3ms/step | 50.0 samples/s
427
+ Step 3110/10001 | Loss: 3.8197 | LR: 3.94e-04 | 1281.7ms/step | 49.9 samples/s
428
+ Step 3120/10001 | Loss: 3.8046 | LR: 3.94e-04 | 1279.8ms/step | 50.0 samples/s
429
+ Step 3130/10001 | Loss: 3.7822 | LR: 3.93e-04 | 1280.2ms/step | 50.0 samples/s
430
+ Step 3140/10001 | Loss: 3.7982 | LR: 3.92e-04 | 1283.9ms/step | 49.8 samples/s
431
+ Step 3150/10001 | Loss: 3.7891 | LR: 3.92e-04 | 1280.1ms/step | 50.0 samples/s
432
+ Step 3160/10001 | Loss: 3.7634 | LR: 3.91e-04 | 1280.1ms/step | 50.0 samples/s
433
+ Step 3170/10001 | Loss: 3.7652 | LR: 3.90e-04 | 1280.4ms/step | 50.0 samples/s
434
+ Step 3180/10001 | Loss: 3.7804 | LR: 3.90e-04 | 1279.7ms/step | 50.0 samples/s
435
+ Step 3190/10001 | Loss: 3.7990 | LR: 3.89e-04 | 1279.9ms/step | 50.0 samples/s
436
+ Step 3200/10001 | Loss: 3.7740 | LR: 3.89e-04 | 1282.4ms/step | 49.9 samples/s
437
+ Step 3210/10001 | Loss: 3.7700 | LR: 3.88e-04 | 1281.4ms/step | 49.9 samples/s
438
+ Step 3220/10001 | Loss: 3.7856 | LR: 3.87e-04 | 1280.4ms/step | 50.0 samples/s
439
+ Step 3230/10001 | Loss: 3.7708 | LR: 3.87e-04 | 1279.7ms/step | 50.0 samples/s
440
+ Step 3240/10001 | Loss: 3.7624 | LR: 3.86e-04 | 1286.0ms/step | 49.8 samples/s
441
+ Step 3250/10001 | Loss: 3.7657 | LR: 3.85e-04 | 1280.1ms/step | 50.0 samples/s
442
+ Validation at step 3250: Loss = 3.7750, Perplexity = 43.60
443
+ Step 3260/10001 | Loss: 3.7816 | LR: 3.85e-04 | 26154.2ms/step | 2.4 samples/s
444
+ Step 3270/10001 | Loss: 3.7725 | LR: 3.84e-04 | 1280.5ms/step | 50.0 samples/s
445
+ Step 3280/10001 | Loss: 3.7795 | LR: 3.83e-04 | 1281.5ms/step | 49.9 samples/s
446
+ Step 3290/10001 | Loss: 3.7576 | LR: 3.83e-04 | 1280.2ms/step | 50.0 samples/s
447
+ Step 3300/10001 | Loss: 3.7815 | LR: 3.82e-04 | 1280.0ms/step | 50.0 samples/s
448
+ Step 3310/10001 | Loss: 3.7696 | LR: 3.81e-04 | 1286.8ms/step | 49.7 samples/s
449
+ Step 3320/10001 | Loss: 3.7566 | LR: 3.80e-04 | 1281.2ms/step | 50.0 samples/s
450
+ Step 3330/10001 | Loss: 3.7327 | LR: 3.80e-04 | 1279.8ms/step | 50.0 samples/s
451
+ Step 3340/10001 | Loss: 3.7452 | LR: 3.79e-04 | 1280.5ms/step | 50.0 samples/s
452
+ Step 3350/10001 | Loss: 3.7428 | LR: 3.78e-04 | 1281.1ms/step | 50.0 samples/s
453
+ Step 3360/10001 | Loss: 3.7494 | LR: 3.78e-04 | 1280.3ms/step | 50.0 samples/s
454
+ Step 3370/10001 | Loss: 3.7525 | LR: 3.77e-04 | 1280.3ms/step | 50.0 samples/s
455
+ Step 3380/10001 | Loss: 3.7423 | LR: 3.76e-04 | 1282.1ms/step | 49.9 samples/s
456
+ Step 3390/10001 | Loss: 3.7513 | LR: 3.76e-04 | 1279.9ms/step | 50.0 samples/s
457
+ Step 3400/10001 | Loss: 3.7291 | LR: 3.75e-04 | 1280.1ms/step | 50.0 samples/s
458
+ Step 3410/10001 | Loss: 3.7546 | LR: 3.74e-04 | 1280.2ms/step | 50.0 samples/s
459
+ Step 3420/10001 | Loss: 3.7451 | LR: 3.74e-04 | 1279.4ms/step | 50.0 samples/s
460
+ Step 3430/10001 | Loss: 3.7342 | LR: 3.73e-04 | 1280.1ms/step | 50.0 samples/s
461
+ Step 3440/10001 | Loss: 3.7350 | LR: 3.72e-04 | 1280.4ms/step | 50.0 samples/s
462
+ Step 3450/10001 | Loss: 3.7621 | LR: 3.72e-04 | 1279.5ms/step | 50.0 samples/s
463
+ Step 3460/10001 | Loss: 3.7535 | LR: 3.71e-04 | 1280.0ms/step | 50.0 samples/s
464
+ Step 3470/10001 | Loss: 3.7288 | LR: 3.70e-04 | 1280.6ms/step | 50.0 samples/s
465
+ Step 3480/10001 | Loss: 3.7326 | LR: 3.69e-04 | 1279.5ms/step | 50.0 samples/s
466
+ Step 3490/10001 | Loss: 3.7382 | LR: 3.69e-04 | 1280.1ms/step | 50.0 samples/s
467
+ Step 3500/10001 | Loss: 3.7157 | LR: 3.68e-04 | 1280.1ms/step | 50.0 samples/s
468
+ Validation at step 3500: Loss = 3.7437, Perplexity = 42.25
469
+ Step 3510/10001 | Loss: 3.7213 | LR: 3.67e-04 | 26148.5ms/step | 2.4 samples/s
epfl_final/wandb/run-20260430_022845-fnhn8tt1/files/requirements.txt ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ menuinst==2.4.2
2
+ anaconda-anon-usage==0.7.6
3
+ annotated-types==0.6.0
4
+ archspec==0.2.5
5
+ boltons==25.0.0
6
+ certifi==2026.4.22
7
+ charset-normalizer==3.4.4
8
+ click==8.2.1
9
+ distro==1.9.0
10
+ frozendict==2.4.6
11
+ h11==0.16.0
12
+ idna==3.11
13
+ jaraco.context==6.1.0
14
+ jeepney==0.7.1
15
+ jsonpointer==3.1.1
16
+ libmambapy==2.3.2
17
+ mdurl==0.1.2
18
+ more-itertools==11.0.2
19
+ msgpack==1.1.1
20
+ packaging==26.0
21
+ pkce==1.0.3
22
+ platformdirs==4.9.4
23
+ pluggy==1.6.0
24
+ pycosat==0.6.6
25
+ pycparser==3.0
26
+ Pygments==2.20.0
27
+ PyJWT==2.12.1
28
+ PySocks==1.7.1
29
+ python-dotenv==1.2.1
30
+ readchar==4.2.1
31
+ ruamel.yaml.clib==0.2.14
32
+ semver==3.0.4
33
+ shellingham==1.5.4
34
+ tomli==2.4.0
35
+ tomlkit==0.13.3
36
+ tqdm==4.67.3
37
+ truststore==0.10.1
38
+ typing_extensions==4.15.0
39
+ anyio==4.12.1
40
+ cffi==1.17.1
41
+ httpcore==1.0.9
42
+ jaraco.classes==3.4.0
43
+ jaraco.functools==4.4.0
44
+ jsonpatch==1.33
45
+ markdown-it-py==4.0.0
46
+ ruamel.yaml==0.18.16
47
+ typer-slim==0.20.0
48
+ typing-inspection==0.4.2
49
+ wheel==0.46.3
50
+ brotlicffi==1.2.0.0
51
+ cryptography==45.0.7
52
+ httpx==0.28.1
53
+ pip==26.0.1
54
+ pydantic_core==2.46.2
55
+ rich==14.2.0
56
+ zstandard==0.25.0
57
+ pydantic==2.13.2
58
+ SecretStorage==3.5.0
59
+ urllib3==2.6.3
60
+ keyring==25.7.0
61
+ pydantic-settings==2.12.0
62
+ requests==2.33.1
63
+ typer==0.20.0
64
+ anaconda-cli-base==0.8.2
65
+ conda_package_streaming==0.12.0
66
+ anaconda-auth==0.14.2
67
+ conda-package-handling==2.4.0
68
+ conda==26.3.2
69
+ conda-anaconda-telemetry==0.3.0
70
+ conda-anaconda-tos==0.2.2
71
+ conda-content-trust==0.3.1
72
+ conda-libmamba-solver==26.4.0
73
+ smmap==5.0.3
74
+ sentry-sdk==2.58.0
75
+ PyYAML==6.0.3
76
+ protobuf==7.34.1
77
+ gitdb==4.0.12
78
+ GitPython==3.1.49
79
+ wandb==0.26.1
80
+ nvidia-cusparselt-cu13==0.8.0
81
+ mpmath==1.3.0
82
+ cuda-toolkit==13.0.2
83
+ xxhash==3.7.0
84
+ triton==3.6.0
85
+ torchinfo==1.8.0
86
+ sympy==1.14.0
87
+ six==1.17.0
88
+ setuptools==81.0.0
89
+ safetensors==0.7.0
90
+ regex==2026.4.4
91
+ pyarrow==24.0.0
92
+ psutil==7.2.2
93
+ propcache==0.4.1
94
+ nvidia-nvtx==13.0.85
95
+ nvidia-nvshmem-cu13==3.4.5
96
+ nvidia-nvjitlink==13.0.88
97
+ nvidia-nccl-cu13==2.28.9
98
+ nvidia-curand==10.4.0.35
99
+ nvidia-cufile==1.15.1.6
100
+ nvidia-cuda-runtime==13.0.96
101
+ nvidia-cuda-nvrtc==13.0.88
102
+ nvidia-cuda-cupti==13.0.85
103
+ nvidia-cublas==13.1.0.3
104
+ numpy==2.4.4
105
+ networkx==3.6.1
106
+ multidict==6.7.1
107
+ MarkupSafe==3.0.3
108
+ hf-xet==1.4.3
109
+ fsspec==2026.2.0
110
+ frozenlist==1.8.0
111
+ filelock==3.29.0
112
+ dill==0.4.1
113
+ cuda-pathfinder==1.5.4
114
+ attrs==26.1.0
115
+ aiohappyeyeballs==2.6.1
116
+ yarl==1.23.0
117
+ python-dateutil==2.9.0.post0
118
+ nvidia-cusparse==12.6.3.3
119
+ nvidia-cufft==12.0.0.61
120
+ nvidia-cudnn-cu13==9.19.0.56
121
+ multiprocess==0.70.19
122
+ Jinja2==3.1.6
123
+ cuda-bindings==13.2.0
124
+ aiosignal==1.4.0
125
+ pandas==3.0.2
126
+ nvidia-cusolver==12.0.4.66
127
+ aiohttp==3.13.5
128
+ huggingface_hub==1.12.2
129
+ torch==2.11.0
130
+ tokenizers==0.22.2
131
+ datasets==4.8.5
132
+ transformers==5.7.0
133
+ accelerate==1.13.0
epfl_final/wandb/run-20260430_022845-fnhn8tt1/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-04-30T02:28:45.322720Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-10-0-0",
11
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6689402109952"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "uckkuns0p3h0lzywnnkwv9osc0va8dhu"
36
+ }
epfl_final/wandb/run-20260430_022845-fnhn8tt1/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260430_022845-fnhn8tt1/logs/debug.log ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-04-30 02:28:45,340 INFO MainThread:93114 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-04-30 02:28:45,341 INFO MainThread:93114 [wandb_setup.py:_flush():81] Configure stats pid to 93114
3
+ 2026-04-30 02:28:45,341 INFO MainThread:93114 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-04-30 02:28:45,341 INFO MainThread:93114 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_022845-fnhn8tt1/logs/debug.log
5
+ 2026-04-30 02:28:45,341 INFO MainThread:93114 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_022845-fnhn8tt1/logs/debug-internal.log
6
+ 2026-04-30 02:28:45,341 INFO MainThread:93114 [wandb_init.py:init():850] calling init triggers
7
+ 2026-04-30 02:28:45,341 INFO MainThread:93114 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 2048, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_022841', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10001, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'flash_attention': True, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 1, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, '_wandb': {}}
9
+ 2026-04-30 02:28:45,341 INFO MainThread:93114 [wandb_init.py:init():898] starting backend
10
+ 2026-04-30 02:28:45,856 INFO MainThread:93114 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-04-30 02:28:46,211 INFO MainThread:93114 [wandb_init.py:init():918] backend started and connected
12
+ 2026-04-30 02:28:46,214 INFO MainThread:93114 [wandb_init.py:init():988] updated telemetry
13
+ 2026-04-30 02:28:46,215 INFO MainThread:93114 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-04-30 02:28:46,768 INFO MainThread:93114 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-04-30 02:28:46,981 INFO MainThread:93114 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-04-30 02:28:46,981 INFO MainThread:93114 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-04-30 02:28:46,981 INFO MainThread:93114 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-04-30 02:28:46,981 INFO MainThread:93114 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-04-30 02:28:46,989 INFO MainThread:93114 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/wandb/run-20260430_022845-fnhn8tt1/run-fnhn8tt1.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:32865ebd44aa7fcc494a4cf16de73bcaf8e6736692fef49b26412d47ba152cb2
3
+ size 1376256
epfl_final/wandb/run-20260430_023222-0isb4snr/files/config.yaml ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ 1qjn2mtlxumo9vgtpjx06t3akijupcu8:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "6689491255296"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /mloscratch/homes/navasard/conda/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41
25
+ host: h100-workspace-10-0-0
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-04-30T02:32:22.609973Z"
33
+ writerId: 1qjn2mtlxumo9vgtpjx06t3akijupcu8
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 49
40
+ - 51
41
+ "2":
42
+ - 1
43
+ - 11
44
+ - 49
45
+ - 51
46
+ - 71
47
+ "3":
48
+ - 15
49
+ - 16
50
+ "4": 3.10.12
51
+ "5": 0.26.1
52
+ "12": 0.26.1
53
+ "13": linux-x86_64
54
+ attention_type:
55
+ value: sparse_stochastic
56
+ checkpoint_activations:
57
+ value: false
58
+ eval_batches:
59
+ value: 500
60
+ eval_steps:
61
+ value: 250
62
+ gradient_accumulation_steps:
63
+ value: 8
64
+ init_from_scratch:
65
+ value: true
66
+ learning_rate:
67
+ value: 0.0005
68
+ logging_steps:
69
+ value: 10
70
+ lr_scheduler:
71
+ value: cosine
72
+ max_grad_norm:
73
+ value: 1
74
+ max_seq_length:
75
+ value: 2048
76
+ max_steps:
77
+ value: 10001
78
+ mixed_precision:
79
+ value: bf16
80
+ model_dtype:
81
+ value: bfloat16
82
+ model_path:
83
+ value: /mloscratch/homes/navasard/qwen3_06b_base
84
+ num_epochs:
85
+ value: 1
86
+ num_workers:
87
+ value: 8
88
+ output_dir:
89
+ value: ./checkpoints/run_20260430_023219
90
+ per_device_batch_size:
91
+ value: 8
92
+ resume_from_checkpoint:
93
+ value: null
94
+ save_steps:
95
+ value: 2500
96
+ seed:
97
+ value: 42
98
+ sparse_block_size:
99
+ value: 128
100
+ sparse_mask_pool_size:
101
+ value: 1000
102
+ sparse_num_random_blocks:
103
+ value: 1
104
+ sparse_val_dense_compare:
105
+ value: false
106
+ strategy:
107
+ value: ddp
108
+ train_path:
109
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
110
+ val_path:
111
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
112
+ verbose:
113
+ value: true
114
+ wandb_enabled:
115
+ value: true
116
+ wandb_name:
117
+ value: null
118
+ wandb_project:
119
+ value: stochastic_sparsity
120
+ wandb_tags:
121
+ value:
122
+ - qwen3
123
+ - 0.6b
124
+ warmup_steps:
125
+ value: 100
126
+ weight_decay:
127
+ value: 0.01
epfl_final/wandb/run-20260430_023222-0isb4snr/files/output.log ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
2
+ Initializing model from scratch (random weights)...
3
+ Checkpoint activations: False
4
+ Traceback (most recent call last):
5
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1043, in <module>
6
+ main()
7
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1039, in main
8
+ train(config)
9
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 651, in train
10
+ model = Qwen3ForCausalLM(model_config)
11
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 237, in __init__
12
+ self.model = Qwen3Model(config)
13
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 105, in __init__
14
+ self.layers = nn.ModuleList([
15
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 106, in <listcomp>
16
+ Qwen3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)
17
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 51, in __init__
18
+ self.self_attn = Qwen3SparseStochasticAttention(config, layer_idx)
19
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 72, in __init__
20
+ self._mask_pool = self._build_mask_pool(torch.device(buffer_device))
21
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 78, in _build_mask_pool
22
+ pool = self._build_mask_pool_inner(device)
23
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 111, in _build_mask_pool_inner
24
+ self._rt_pool[i, qb] = perm.to(dtype=self._rt_pool.dtype)
25
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1955, in __getattr__
26
+ def __getattr__(self, name: str) -> Union[Tensor, "Module"]:
27
+ KeyboardInterrupt
28
+ Traceback (most recent call last):
29
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1043, in <module>
30
+ main()
31
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1039, in main
32
+ train(config)
33
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 651, in train
34
+ model = Qwen3ForCausalLM(model_config)
35
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 237, in __init__
36
+ self.model = Qwen3Model(config)
37
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 105, in __init__
38
+ self.layers = nn.ModuleList([
39
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 106, in <listcomp>
40
+ Qwen3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)
41
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/model.py", line 51, in __init__
42
+ self.self_attn = Qwen3SparseStochasticAttention(config, layer_idx)
43
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 72, in __init__
44
+ self._mask_pool = self._build_mask_pool(torch.device(buffer_device))
45
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 78, in _build_mask_pool
46
+ pool = self._build_mask_pool_inner(device)
47
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/sparse_attention.py", line 111, in _build_mask_pool_inner
48
+ self._rt_pool[i, qb] = perm.to(dtype=self._rt_pool.dtype)
49
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1955, in __getattr__
50
+ def __getattr__(self, name: str) -> Union[Tensor, "Module"]:
51
+ KeyboardInterrupt
epfl_final/wandb/run-20260430_023222-0isb4snr/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.1.post1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.4.3
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.58.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.3
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.3
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.49
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.0
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.12.2
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.7.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260430_023222-0isb4snr/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-04-30T02:32:22.609973Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-10-0-0",
11
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6689491255296"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-069bdfaf-ef1c-f7f2-f9e3-2249bb9dfc41"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "1qjn2mtlxumo9vgtpjx06t3akijupcu8"
36
+ }
epfl_final/wandb/run-20260430_023222-0isb4snr/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb":{"runtime":15},"_runtime":15}
epfl_final/wandb/run-20260430_023222-0isb4snr/logs/debug-internal.log ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-04-30T02:32:22.869922Z","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-04-30T02:32:22.870420284Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
3
+ {"time":"2026-04-30T02:32:23.174024959Z","level":"INFO","msg":"stream: created new stream","id":"0isb4snr"}
4
+ {"time":"2026-04-30T02:32:23.174134452Z","level":"INFO","msg":"handler: started"}
5
+ {"time":"2026-04-30T02:32:23.176643239Z","level":"INFO","msg":"stream: started"}
6
+ {"time":"2026-04-30T02:32:23.176680896Z","level":"INFO","msg":"writer: started","stream_id":"0isb4snr"}
7
+ {"time":"2026-04-30T02:32:23.176733725Z","level":"INFO","msg":"sender: started"}
8
+ {"time":"2026-04-30T02:32:23.907221489Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
9
+ {"time":"2026-04-30T02:32:24.209435667Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-04-30T02:32:38.90721036Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":3,"uploaded_len":2}
11
+ {"time":"2026-04-30T02:32:39.213994744Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-04-30T02:32:39.867923032Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
13
+ {"time":"2026-04-30T02:32:39.86821621Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":3,"console_lines":48,"uploaded_len":3,"complete":true,"exit_code":255}
14
+ {"time":"2026-04-30T02:32:40.157464582Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
15
+ {"time":"2026-04-30T02:32:40.157673695Z","level":"INFO","msg":"stream: finishing up"}
16
+ {"time":"2026-04-30T02:32:40.157734997Z","level":"INFO","msg":"handler: closed"}
17
+ {"time":"2026-04-30T02:32:40.159021378Z","level":"INFO","msg":"sender: closed"}
18
+ {"time":"2026-04-30T02:32:40.159033185Z","level":"INFO","msg":"stream: all finished"}
epfl_final/wandb/run-20260430_023222-0isb4snr/logs/debug.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-04-30 02:32:22,627 INFO MainThread:95243 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-04-30 02:32:22,627 INFO MainThread:95243 [wandb_setup.py:_flush():81] Configure stats pid to 95243
3
+ 2026-04-30 02:32:22,627 INFO MainThread:95243 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-04-30 02:32:22,627 INFO MainThread:95243 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_023222-0isb4snr/logs/debug.log
5
+ 2026-04-30 02:32:22,627 INFO MainThread:95243 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_023222-0isb4snr/logs/debug-internal.log
6
+ 2026-04-30 02:32:22,628 INFO MainThread:95243 [wandb_init.py:init():850] calling init triggers
7
+ 2026-04-30 02:32:22,628 INFO MainThread:95243 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 2048, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_023219', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10001, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 1, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, '_wandb': {}}
9
+ 2026-04-30 02:32:22,628 INFO MainThread:95243 [wandb_init.py:init():898] starting backend
10
+ 2026-04-30 02:32:22,857 INFO MainThread:95243 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-04-30 02:32:23,177 INFO MainThread:95243 [wandb_init.py:init():918] backend started and connected
12
+ 2026-04-30 02:32:23,179 INFO MainThread:95243 [wandb_init.py:init():988] updated telemetry
13
+ 2026-04-30 02:32:23,180 INFO MainThread:95243 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-04-30 02:32:23,819 INFO MainThread:95243 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-04-30 02:32:23,903 INFO MainThread:95243 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-04-30 02:32:23,903 INFO MainThread:95243 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-04-30 02:32:23,903 INFO MainThread:95243 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-04-30 02:32:23,903 INFO MainThread:95243 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-04-30 02:32:23,905 INFO MainThread:95243 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-04-30 02:32:38,995 INFO wandb-AsyncioManager-main:95243 [service_client.py:_forward_responses():134] Reached EOF.
21
+ 2026-04-30 02:32:38,996 INFO wandb-AsyncioManager-main:95243 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
epfl_final/wandb/run-20260430_023222-0isb4snr/run-0isb4snr.wandb ADDED
Binary file (7.95 kB). View file
 
epfl_final/wandb/run-20260430_153400-3fhhow4v/files/config.yaml ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ hrr9zq6haocwqq6tjzwaw1cgpg3n7jv2:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "6713334460416"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /mloscratch/homes/navasard/conda/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7
25
+ host: h100-workspace-10-0-6
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.13.13
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-04-30T15:34:00.733112Z"
33
+ writerId: hrr9zq6haocwqq6tjzwaw1cgpg3n7jv2
34
+ m: []
35
+ python_version: 3.13.13
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 49
40
+ - 51
41
+ "2":
42
+ - 1
43
+ - 11
44
+ - 49
45
+ - 51
46
+ - 71
47
+ "3":
48
+ - 2
49
+ - 15
50
+ - 16
51
+ - 61
52
+ "4": 3.13.13
53
+ "5": 0.26.1
54
+ "12": 0.26.1
55
+ "13": linux-x86_64
56
+ attention_type:
57
+ value: sparse_stochastic
58
+ checkpoint_activations:
59
+ value: false
60
+ eval_batches:
61
+ value: 500
62
+ eval_steps:
63
+ value: 250
64
+ gradient_accumulation_steps:
65
+ value: 8
66
+ init_from_scratch:
67
+ value: true
68
+ learning_rate:
69
+ value: 0.0005
70
+ logging_steps:
71
+ value: 10
72
+ lr_scheduler:
73
+ value: cosine
74
+ max_grad_norm:
75
+ value: 1
76
+ max_seq_length:
77
+ value: 3840
78
+ max_steps:
79
+ value: 10005
80
+ mixed_precision:
81
+ value: bf16
82
+ model_dtype:
83
+ value: bfloat16
84
+ model_path:
85
+ value: /mloscratch/homes/navasard/qwen3_06b_base
86
+ num_epochs:
87
+ value: 1
88
+ num_workers:
89
+ value: 8
90
+ output_dir:
91
+ value: ./checkpoints/run_20260430_153357
92
+ per_device_batch_size:
93
+ value: 8
94
+ resume_from_checkpoint:
95
+ value: null
96
+ save_steps:
97
+ value: 2500
98
+ seed:
99
+ value: 42
100
+ sparse_block_size:
101
+ value: 128
102
+ sparse_layer_indices:
103
+ value: null
104
+ sparse_mask_pool_size:
105
+ value: 1000
106
+ sparse_num_random_blocks:
107
+ value: 9
108
+ sparse_val_dense_compare:
109
+ value: false
110
+ strategy:
111
+ value: ddp
112
+ train_path:
113
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
114
+ val_path:
115
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
116
+ verbose:
117
+ value: true
118
+ wandb_enabled:
119
+ value: true
120
+ wandb_name:
121
+ value: null
122
+ wandb_project:
123
+ value: stochastic_sparsity
124
+ wandb_tags:
125
+ value:
126
+ - qwen3
127
+ - 0.6b
128
+ warmup_steps:
129
+ value: 100
130
+ weight_decay:
131
+ value: 0.01
epfl_final/wandb/run-20260430_153400-3fhhow4v/files/output.log ADDED
@@ -0,0 +1,611 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
2
+ Initializing model from scratch (random weights)...
3
+ Checkpoint activations: False
4
+
5
+
6
+ ================================================================================
7
+ MODEL ARCHITECTURE
8
+ ================================================================================
9
+ Model config:
10
+ vocab_size: 151936
11
+ hidden_size: 1024
12
+ num_hidden_layers: 28
13
+ num_attention_heads: 16
14
+ num_key_value_heads: 8
15
+ head_dim: 128
16
+ intermediate_size: 3072
17
+ max_position_embeddings: 32768
18
+ rope_theta: 1000000
19
+ attention_type: sparse_stochastic
20
+ sparse_block_size: 128
21
+ sparse_num_random_blocks: 9
22
+ sparse_mask_pool_size: 1000
23
+ sparse_max_seq_length: 3840
24
+
25
+ Parameter count:
26
+ Total parameters: 596,049,920
27
+ Trainable parameters: 596,049,920
28
+ Non-trainable parameters: 0
29
+ Model size (BF16): 1.19 GB
30
+
31
+ --------------------------------------------------------------------------------
32
+ LAYER-BY-LAYER SUMMARY
33
+ --------------------------------------------------------------------------------
34
+ /mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
35
+
36
+ SOLUTION: Use torch.compile(flex_attention)(...)
37
+
38
+ If you want to debug your score_mod/mask_mod, you can set:
39
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
40
+
41
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
42
+ _warn_once(
43
+ ================================================================================================================================================================
44
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
45
+ ================================================================================================================================================================
46
+ Qwen3ForCausalLM [8, 3840] -- -- True
47
+ ├─Qwen3Model: 1-1 -- [8, 3840, 1024] -- True
48
+ │ └─Embedding: 2-1 [8, 3840] [8, 3840, 1024] 155,582,464 True
49
+ │ └─ModuleList: 2-2 -- -- -- True
50
+ │ │ └─Qwen3DecoderLayer: 3-1 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-2 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-3 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-4 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-5 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-6 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-7 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-8 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-9 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-10 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-11 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-12 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-13 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-14 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-15 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-16 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-17 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-18 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-19 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-20 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-21 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-22 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-23 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-24 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-25 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
75
+ │ │ └─Qwen3DecoderLayer: 3-26 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
76
+ │ │ └─Qwen3DecoderLayer: 3-27 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
77
+ │ │ └─Qwen3DecoderLayer: 3-28 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
78
+ │ └─Qwen3RMSNorm: 2-3 [8, 3840, 1024] [8, 3840, 1024] 1,024 True
79
+ ├─Linear: 1-2 [8, 3840, 1024] [8, 3840, 151936] 155,582,464 True
80
+ ================================================================================================================================================================
81
+ Total params: 751,632,384
82
+ Trainable params: 751,632,384
83
+ Non-trainable params: 0
84
+ Total mult-adds (Units.GIGABYTES): 6.01
85
+ ================================================================================================================================================================
86
+ Input size (MB): 0.25
87
+ Forward/backward pass size (MB): 78816.22
88
+ Params size (MB): 1503.26
89
+ Estimated Total Size (MB): 80319.73
90
+ ================================================================================================================================================================
91
+ ================================================================================
92
+
93
+ Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
94
+ Using pre-tokenized dataset (no runtime tokenization)
95
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
96
+ Loaded 781148 pre-tokenized examples (stored_len=2048, pack_factor=2, packed_examples=390574)
97
+ Dataset split: 312459 train, 78115 validation
98
+ Training examples: 312459
99
+ Validation examples: 78115
100
+ Steps per epoch: 4882
101
+ Total training steps: 10005
102
+ Starting training...
103
+ Step 10/10005 | Loss: 11.8553 | LR: 5.00e-05 | 7584.8ms/step | 8.4 samples/s
104
+ Step 20/10005 | Loss: 10.6364 | LR: 1.00e-04 | 2641.7ms/step | 24.2 samples/s
105
+ Step 30/10005 | Loss: 9.6533 | LR: 1.50e-04 | 2641.0ms/step | 24.2 samples/s
106
+ Step 40/10005 | Loss: 8.5269 | LR: 2.00e-04 | 2638.9ms/step | 24.3 samples/s
107
+ Step 50/10005 | Loss: 7.9860 | LR: 2.50e-04 | 2645.2ms/step | 24.2 samples/s
108
+ Step 60/10005 | Loss: 7.8369 | LR: 3.00e-04 | 2650.4ms/step | 24.1 samples/s
109
+ Step 70/10005 | Loss: 7.6571 | LR: 3.50e-04 | 2658.8ms/step | 24.1 samples/s
110
+ Step 80/10005 | Loss: 7.4166 | LR: 4.00e-04 | 2669.5ms/step | 24.0 samples/s
111
+ Step 90/10005 | Loss: 7.2103 | LR: 4.50e-04 | 2661.7ms/step | 24.0 samples/s
112
+ Step 100/10005 | Loss: 7.0364 | LR: 5.00e-04 | 2663.4ms/step | 24.0 samples/s
113
+ Step 110/10005 | Loss: 6.9072 | LR: 5.00e-04 | 2662.8ms/step | 24.0 samples/s
114
+ Step 120/10005 | Loss: 6.8022 | LR: 5.00e-04 | 2662.6ms/step | 24.0 samples/s
115
+ Step 130/10005 | Loss: 6.7570 | LR: 5.00e-04 | 2665.1ms/step | 24.0 samples/s
116
+ Step 140/10005 | Loss: 6.6367 | LR: 5.00e-04 | 2667.4ms/step | 24.0 samples/s
117
+ Step 150/10005 | Loss: 6.5516 | LR: 5.00e-04 | 2667.7ms/step | 24.0 samples/s
118
+ Step 160/10005 | Loss: 6.4725 | LR: 5.00e-04 | 2665.3ms/step | 24.0 samples/s
119
+ Step 170/10005 | Loss: 6.3641 | LR: 5.00e-04 | 2662.9ms/step | 24.0 samples/s
120
+ Step 180/10005 | Loss: 6.3385 | LR: 5.00e-04 | 2662.8ms/step | 24.0 samples/s
121
+ Step 190/10005 | Loss: 6.2407 | LR: 5.00e-04 | 2665.9ms/step | 24.0 samples/s
122
+ Step 200/10005 | Loss: 6.1735 | LR: 5.00e-04 | 2666.5ms/step | 24.0 samples/s
123
+ Step 210/10005 | Loss: 6.1931 | LR: 5.00e-04 | 2665.3ms/step | 24.0 samples/s
124
+ Step 220/10005 | Loss: 6.0943 | LR: 5.00e-04 | 2665.3ms/step | 24.0 samples/s
125
+ Step 230/10005 | Loss: 5.9786 | LR: 5.00e-04 | 2664.0ms/step | 24.0 samples/s
126
+ Step 240/10005 | Loss: 5.9752 | LR: 5.00e-04 | 2662.0ms/step | 24.0 samples/s
127
+ Step 250/10005 | Loss: 5.9291 | LR: 5.00e-04 | 2662.5ms/step | 24.0 samples/s
128
+ Validation at step 250: Loss = 5.8606, Perplexity = 350.94
129
+ Step 260/10005 | Loss: 5.8437 | LR: 5.00e-04 | 53646.8ms/step | 1.2 samples/s
130
+ Step 270/10005 | Loss: 5.7943 | LR: 5.00e-04 | 2669.5ms/step | 24.0 samples/s
131
+ Step 280/10005 | Loss: 5.7501 | LR: 5.00e-04 | 2666.2ms/step | 24.0 samples/s
132
+ Step 290/10005 | Loss: 5.7110 | LR: 5.00e-04 | 2666.9ms/step | 24.0 samples/s
133
+ Step 300/10005 | Loss: 5.6611 | LR: 4.99e-04 | 2667.3ms/step | 24.0 samples/s
134
+ Step 310/10005 | Loss: 5.5797 | LR: 4.99e-04 | 2667.2ms/step | 24.0 samples/s
135
+ Step 320/10005 | Loss: 5.5611 | LR: 4.99e-04 | 2669.8ms/step | 24.0 samples/s
136
+ Step 330/10005 | Loss: 5.5107 | LR: 4.99e-04 | 2666.8ms/step | 24.0 samples/s
137
+ Step 340/10005 | Loss: 5.4566 | LR: 4.99e-04 | 2670.4ms/step | 24.0 samples/s
138
+ Step 350/10005 | Loss: 5.4273 | LR: 4.99e-04 | 2668.7ms/step | 24.0 samples/s
139
+ Step 360/10005 | Loss: 5.4127 | LR: 4.99e-04 | 2672.9ms/step | 23.9 samples/s
140
+ Step 370/10005 | Loss: 5.3447 | LR: 4.99e-04 | 2666.2ms/step | 24.0 samples/s
141
+ Step 380/10005 | Loss: 5.2756 | LR: 4.99e-04 | 2673.0ms/step | 23.9 samples/s
142
+ Step 390/10005 | Loss: 5.2364 | LR: 4.99e-04 | 2671.6ms/step | 24.0 samples/s
143
+ Step 400/10005 | Loss: 5.1967 | LR: 4.99e-04 | 2669.4ms/step | 24.0 samples/s
144
+ Step 410/10005 | Loss: 5.1558 | LR: 4.99e-04 | 2667.7ms/step | 24.0 samples/s
145
+ Step 420/10005 | Loss: 5.0870 | LR: 4.99e-04 | 2667.3ms/step | 24.0 samples/s
146
+ Step 430/10005 | Loss: 5.0922 | LR: 4.99e-04 | 2669.0ms/step | 24.0 samples/s
147
+ Step 440/10005 | Loss: 5.0433 | LR: 4.99e-04 | 2669.2ms/step | 24.0 samples/s
148
+ Step 450/10005 | Loss: 5.0095 | LR: 4.98e-04 | 2665.4ms/step | 24.0 samples/s
149
+ Step 460/10005 | Loss: 4.9786 | LR: 4.98e-04 | 2662.9ms/step | 24.0 samples/s
150
+ Step 470/10005 | Loss: 4.9332 | LR: 4.98e-04 | 2663.6ms/step | 24.0 samples/s
151
+ Step 480/10005 | Loss: 4.9080 | LR: 4.98e-04 | 2667.9ms/step | 24.0 samples/s
152
+ Step 490/10005 | Loss: 4.8520 | LR: 4.98e-04 | 2667.6ms/step | 24.0 samples/s
153
+ Step 500/10005 | Loss: 4.8217 | LR: 4.98e-04 | 2667.5ms/step | 24.0 samples/s
154
+ Validation at step 500: Loss = 4.8431, Perplexity = 126.87
155
+ Step 510/10005 | Loss: 4.8210 | LR: 4.98e-04 | 51472.9ms/step | 1.2 samples/s
156
+ Step 520/10005 | Loss: 4.7895 | LR: 4.98e-04 | 2666.1ms/step | 24.0 samples/s
157
+ Step 530/10005 | Loss: 4.7717 | LR: 4.98e-04 | 2667.2ms/step | 24.0 samples/s
158
+ Step 540/10005 | Loss: 4.7398 | LR: 4.98e-04 | 2666.4ms/step | 24.0 samples/s
159
+ Step 550/10005 | Loss: 4.7091 | LR: 4.97e-04 | 2667.1ms/step | 24.0 samples/s
160
+ Step 560/10005 | Loss: 4.6820 | LR: 4.97e-04 | 2666.0ms/step | 24.0 samples/s
161
+ Step 570/10005 | Loss: 4.6714 | LR: 4.97e-04 | 2664.7ms/step | 24.0 samples/s
162
+ Step 580/10005 | Loss: 4.6575 | LR: 4.97e-04 | 2666.8ms/step | 24.0 samples/s
163
+ Step 590/10005 | Loss: 4.6066 | LR: 4.97e-04 | 2664.4ms/step | 24.0 samples/s
164
+ Step 600/10005 | Loss: 4.5943 | LR: 4.97e-04 | 2666.1ms/step | 24.0 samples/s
165
+ Step 610/10005 | Loss: 4.5709 | LR: 4.97e-04 | 2663.2ms/step | 24.0 samples/s
166
+ Step 620/10005 | Loss: 4.5857 | LR: 4.97e-04 | 2662.7ms/step | 24.0 samples/s
167
+ Step 630/10005 | Loss: 4.5633 | LR: 4.96e-04 | 2662.9ms/step | 24.0 samples/s
168
+ Step 640/10005 | Loss: 4.5159 | LR: 4.96e-04 | 2665.8ms/step | 24.0 samples/s
169
+ Step 650/10005 | Loss: 4.4985 | LR: 4.96e-04 | 2660.7ms/step | 24.1 samples/s
170
+ Step 660/10005 | Loss: 4.4937 | LR: 4.96e-04 | 2663.5ms/step | 24.0 samples/s
171
+ Step 670/10005 | Loss: 4.4725 | LR: 4.96e-04 | 2662.5ms/step | 24.0 samples/s
172
+ Step 680/10005 | Loss: 4.4560 | LR: 4.96e-04 | 2662.1ms/step | 24.0 samples/s
173
+ Step 690/10005 | Loss: 4.4397 | LR: 4.96e-04 | 2664.8ms/step | 24.0 samples/s
174
+ Step 700/10005 | Loss: 4.4394 | LR: 4.95e-04 | 2662.7ms/step | 24.0 samples/s
175
+ Step 710/10005 | Loss: 4.4252 | LR: 4.95e-04 | 2664.4ms/step | 24.0 samples/s
176
+ Step 720/10005 | Loss: 4.3984 | LR: 4.95e-04 | 2663.0ms/step | 24.0 samples/s
177
+ Step 730/10005 | Loss: 4.3772 | LR: 4.95e-04 | 2663.2ms/step | 24.0 samples/s
178
+ Step 740/10005 | Loss: 4.3946 | LR: 4.95e-04 | 2664.3ms/step | 24.0 samples/s
179
+ Step 750/10005 | Loss: 4.3532 | LR: 4.95e-04 | 2667.1ms/step | 24.0 samples/s
180
+ Validation at step 750: Loss = 4.3481, Perplexity = 77.33
181
+ Step 760/10005 | Loss: 4.3175 | LR: 4.95e-04 | 51478.7ms/step | 1.2 samples/s
182
+ Step 770/10005 | Loss: 4.3333 | LR: 4.94e-04 | 2667.5ms/step | 24.0 samples/s
183
+ Step 780/10005 | Loss: 4.3192 | LR: 4.94e-04 | 2667.1ms/step | 24.0 samples/s
184
+ Step 790/10005 | Loss: 4.2956 | LR: 4.94e-04 | 2666.2ms/step | 24.0 samples/s
185
+ Step 800/10005 | Loss: 4.2856 | LR: 4.94e-04 | 2666.1ms/step | 24.0 samples/s
186
+ Step 810/10005 | Loss: 4.2752 | LR: 4.94e-04 | 2666.9ms/step | 24.0 samples/s
187
+ Step 820/10005 | Loss: 4.2761 | LR: 4.94e-04 | 2663.7ms/step | 24.0 samples/s
188
+ Step 830/10005 | Loss: 4.2598 | LR: 4.93e-04 | 2666.2ms/step | 24.0 samples/s
189
+ Step 840/10005 | Loss: 4.2413 | LR: 4.93e-04 | 2665.1ms/step | 24.0 samples/s
190
+ Step 850/10005 | Loss: 4.2406 | LR: 4.93e-04 | 2664.8ms/step | 24.0 samples/s
191
+ Step 860/10005 | Loss: 4.2276 | LR: 4.93e-04 | 2662.7ms/step | 24.0 samples/s
192
+ Step 870/10005 | Loss: 4.2239 | LR: 4.93e-04 | 2665.2ms/step | 24.0 samples/s
193
+ Step 880/10005 | Loss: 4.2025 | LR: 4.92e-04 | 2661.2ms/step | 24.0 samples/s
194
+ Step 890/10005 | Loss: 4.2102 | LR: 4.92e-04 | 2663.5ms/step | 24.0 samples/s
195
+ Step 900/10005 | Loss: 4.1703 | LR: 4.92e-04 | 2666.0ms/step | 24.0 samples/s
196
+ Step 910/10005 | Loss: 4.1597 | LR: 4.92e-04 | 2666.3ms/step | 24.0 samples/s
197
+ Step 920/10005 | Loss: 4.1814 | LR: 4.92e-04 | 2665.9ms/step | 24.0 samples/s
198
+ Step 930/10005 | Loss: 4.1644 | LR: 4.91e-04 | 2666.4ms/step | 24.0 samples/s
199
+ Step 940/10005 | Loss: 4.1400 | LR: 4.91e-04 | 2665.7ms/step | 24.0 samples/s
200
+ Step 950/10005 | Loss: 4.1441 | LR: 4.91e-04 | 2667.9ms/step | 24.0 samples/s
201
+ Step 960/10005 | Loss: 4.1218 | LR: 4.91e-04 | 2667.3ms/step | 24.0 samples/s
202
+ Step 970/10005 | Loss: 4.1171 | LR: 4.91e-04 | 2666.3ms/step | 24.0 samples/s
203
+ Step 980/10005 | Loss: 4.1264 | LR: 4.90e-04 | 2663.2ms/step | 24.0 samples/s
204
+ Step 990/10005 | Loss: 4.0956 | LR: 4.90e-04 | 2665.4ms/step | 24.0 samples/s
205
+ Step 1000/10005 | Loss: 4.1099 | LR: 4.90e-04 | 2665.7ms/step | 24.0 samples/s
206
+ Validation at step 1000: Loss = 4.0924, Perplexity = 59.88
207
+ Step 1010/10005 | Loss: 4.0752 | LR: 4.90e-04 | 51493.5ms/step | 1.2 samples/s
208
+ Step 1020/10005 | Loss: 4.0905 | LR: 4.89e-04 | 2666.1ms/step | 24.0 samples/s
209
+ Step 1030/10005 | Loss: 4.0616 | LR: 4.89e-04 | 2665.7ms/step | 24.0 samples/s
210
+ Step 1040/10005 | Loss: 4.0692 | LR: 4.89e-04 | 2661.5ms/step | 24.0 samples/s
211
+ Step 1050/10005 | Loss: 4.0664 | LR: 4.89e-04 | 2660.1ms/step | 24.1 samples/s
212
+ Step 1060/10005 | Loss: 4.0326 | LR: 4.89e-04 | 2663.4ms/step | 24.0 samples/s
213
+ Step 1070/10005 | Loss: 4.0299 | LR: 4.88e-04 | 2664.5ms/step | 24.0 samples/s
214
+ Step 1080/10005 | Loss: 4.0462 | LR: 4.88e-04 | 2666.0ms/step | 24.0 samples/s
215
+ Step 1090/10005 | Loss: 4.0498 | LR: 4.88e-04 | 2665.1ms/step | 24.0 samples/s
216
+ Step 1100/10005 | Loss: 4.0012 | LR: 4.88e-04 | 2667.3ms/step | 24.0 samples/s
217
+ Step 1110/10005 | Loss: 4.0327 | LR: 4.87e-04 | 2664.8ms/step | 24.0 samples/s
218
+ Step 1120/10005 | Loss: 4.0139 | LR: 4.87e-04 | 2664.1ms/step | 24.0 samples/s
219
+ Step 1130/10005 | Loss: 3.9818 | LR: 4.87e-04 | 2665.7ms/step | 24.0 samples/s
220
+ Step 1140/10005 | Loss: 3.9858 | LR: 4.87e-04 | 2665.6ms/step | 24.0 samples/s
221
+ Step 1150/10005 | Loss: 3.9790 | LR: 4.86e-04 | 2664.6ms/step | 24.0 samples/s
222
+ Step 1160/10005 | Loss: 3.9840 | LR: 4.86e-04 | 2664.9ms/step | 24.0 samples/s
223
+ Step 1170/10005 | Loss: 3.9928 | LR: 4.86e-04 | 2666.3ms/step | 24.0 samples/s
224
+ Step 1180/10005 | Loss: 3.9570 | LR: 4.85e-04 | 2666.4ms/step | 24.0 samples/s
225
+ Step 1190/10005 | Loss: 3.9745 | LR: 4.85e-04 | 2666.4ms/step | 24.0 samples/s
226
+ Step 1200/10005 | Loss: 3.9355 | LR: 4.85e-04 | 2663.7ms/step | 24.0 samples/s
227
+ Step 1210/10005 | Loss: 3.9581 | LR: 4.85e-04 | 2663.3ms/step | 24.0 samples/s
228
+ Step 1220/10005 | Loss: 3.9505 | LR: 4.84e-04 | 2664.6ms/step | 24.0 samples/s
229
+ Step 1230/10005 | Loss: 3.9567 | LR: 4.84e-04 | 2665.9ms/step | 24.0 samples/s
230
+ Step 1240/10005 | Loss: 3.9294 | LR: 4.84e-04 | 2663.4ms/step | 24.0 samples/s
231
+ Step 1250/10005 | Loss: 3.9399 | LR: 4.84e-04 | 2662.1ms/step | 24.0 samples/s
232
+ Validation at step 1250: Loss = 3.9333, Perplexity = 51.08
233
+ Step 1260/10005 | Loss: 3.9534 | LR: 4.83e-04 | 51475.6ms/step | 1.2 samples/s
234
+ Step 1270/10005 | Loss: 3.9011 | LR: 4.83e-04 | 2667.4ms/step | 24.0 samples/s
235
+ Step 1280/10005 | Loss: 3.9030 | LR: 4.83e-04 | 2665.8ms/step | 24.0 samples/s
236
+ Step 1290/10005 | Loss: 3.9089 | LR: 4.82e-04 | 2662.2ms/step | 24.0 samples/s
237
+ Step 1300/10005 | Loss: 3.9179 | LR: 4.82e-04 | 2664.6ms/step | 24.0 samples/s
238
+ Step 1310/10005 | Loss: 3.9114 | LR: 4.82e-04 | 2664.0ms/step | 24.0 samples/s
239
+ Step 1320/10005 | Loss: 3.8940 | LR: 4.82e-04 | 2661.6ms/step | 24.0 samples/s
240
+ Step 1330/10005 | Loss: 3.9189 | LR: 4.81e-04 | 2663.8ms/step | 24.0 samples/s
241
+ Step 1340/10005 | Loss: 3.8851 | LR: 4.81e-04 | 2664.9ms/step | 24.0 samples/s
242
+ Step 1350/10005 | Loss: 3.8782 | LR: 4.81e-04 | 2662.1ms/step | 24.0 samples/s
243
+ Step 1360/10005 | Loss: 3.8935 | LR: 4.80e-04 | 2662.8ms/step | 24.0 samples/s
244
+ Step 1370/10005 | Loss: 3.8884 | LR: 4.80e-04 | 2662.8ms/step | 24.0 samples/s
245
+ Step 1380/10005 | Loss: 3.8932 | LR: 4.80e-04 | 2667.0ms/step | 24.0 samples/s
246
+ Step 1390/10005 | Loss: 3.8512 | LR: 4.79e-04 | 2665.4ms/step | 24.0 samples/s
247
+ Step 1400/10005 | Loss: 3.8714 | LR: 4.79e-04 | 2665.2ms/step | 24.0 samples/s
248
+ Step 1410/10005 | Loss: 3.8418 | LR: 4.79e-04 | 2668.0ms/step | 24.0 samples/s
249
+ Step 1420/10005 | Loss: 3.8526 | LR: 4.78e-04 | 2664.6ms/step | 24.0 samples/s
250
+ Step 1430/10005 | Loss: 3.8532 | LR: 4.78e-04 | 2664.6ms/step | 24.0 samples/s
251
+ Step 1440/10005 | Loss: 3.8497 | LR: 4.78e-04 | 2662.2ms/step | 24.0 samples/s
252
+ Step 1450/10005 | Loss: 3.8412 | LR: 4.77e-04 | 2664.4ms/step | 24.0 samples/s
253
+ Step 1460/10005 | Loss: 3.8460 | LR: 4.77e-04 | 2663.8ms/step | 24.0 samples/s
254
+ Step 1470/10005 | Loss: 3.8562 | LR: 4.77e-04 | 2665.8ms/step | 24.0 samples/s
255
+ Step 1480/10005 | Loss: 3.8415 | LR: 4.76e-04 | 2666.4ms/step | 24.0 samples/s
256
+ Step 1490/10005 | Loss: 3.8154 | LR: 4.76e-04 | 2668.7ms/step | 24.0 samples/s
257
+ Step 1500/10005 | Loss: 3.8355 | LR: 4.76e-04 | 2663.1ms/step | 24.0 samples/s
258
+ Validation at step 1500: Loss = 3.8262, Perplexity = 45.89
259
+ Step 1510/10005 | Loss: 3.8378 | LR: 4.75e-04 | 51441.9ms/step | 1.2 samples/s
260
+ Step 1520/10005 | Loss: 3.8020 | LR: 4.75e-04 | 2666.7ms/step | 24.0 samples/s
261
+ Step 1530/10005 | Loss: 3.8160 | LR: 4.75e-04 | 2666.4ms/step | 24.0 samples/s
262
+ Step 1540/10005 | Loss: 3.8178 | LR: 4.74e-04 | 2661.0ms/step | 24.1 samples/s
263
+ Step 1550/10005 | Loss: 3.8147 | LR: 4.74e-04 | 2663.8ms/step | 24.0 samples/s
264
+ Step 1560/10005 | Loss: 3.8158 | LR: 4.74e-04 | 2663.7ms/step | 24.0 samples/s
265
+ Step 1570/10005 | Loss: 3.8260 | LR: 4.73e-04 | 2660.1ms/step | 24.1 samples/s
266
+ Step 1580/10005 | Loss: 3.7739 | LR: 4.73e-04 | 2662.8ms/step | 24.0 samples/s
267
+ Step 1590/10005 | Loss: 3.7890 | LR: 4.73e-04 | 2663.4ms/step | 24.0 samples/s
268
+ Step 1600/10005 | Loss: 3.7605 | LR: 4.72e-04 | 2661.9ms/step | 24.0 samples/s
269
+ Step 1610/10005 | Loss: 3.7898 | LR: 4.72e-04 | 2663.7ms/step | 24.0 samples/s
270
+ Step 1620/10005 | Loss: 3.7812 | LR: 4.72e-04 | 2665.3ms/step | 24.0 samples/s
271
+ Step 1630/10005 | Loss: 3.7955 | LR: 4.71e-04 | 2664.8ms/step | 24.0 samples/s
272
+ Step 1640/10005 | Loss: 3.7678 | LR: 4.71e-04 | 2662.5ms/step | 24.0 samples/s
273
+ Step 1650/10005 | Loss: 3.7796 | LR: 4.70e-04 | 2661.4ms/step | 24.0 samples/s
274
+ Step 1660/10005 | Loss: 3.7692 | LR: 4.70e-04 | 2662.0ms/step | 24.0 samples/s
275
+ Step 1670/10005 | Loss: 3.7597 | LR: 4.70e-04 | 2663.4ms/step | 24.0 samples/s
276
+ Step 1680/10005 | Loss: 3.7687 | LR: 4.69e-04 | 2664.4ms/step | 24.0 samples/s
277
+ Step 1690/10005 | Loss: 3.7680 | LR: 4.69e-04 | 2664.2ms/step | 24.0 samples/s
278
+ Step 1700/10005 | Loss: 3.7391 | LR: 4.68e-04 | 2664.1ms/step | 24.0 samples/s
279
+ Step 1710/10005 | Loss: 3.7253 | LR: 4.68e-04 | 2664.5ms/step | 24.0 samples/s
280
+ Step 1720/10005 | Loss: 3.7555 | LR: 4.68e-04 | 2664.7ms/step | 24.0 samples/s
281
+ Step 1730/10005 | Loss: 3.7409 | LR: 4.67e-04 | 2666.4ms/step | 24.0 samples/s
282
+ Step 1740/10005 | Loss: 3.7282 | LR: 4.67e-04 | 2664.5ms/step | 24.0 samples/s
283
+ Step 1750/10005 | Loss: 3.7111 | LR: 4.67e-04 | 2665.2ms/step | 24.0 samples/s
284
+ Validation at step 1750: Loss = 3.7433, Perplexity = 42.24
285
+ Step 1760/10005 | Loss: 3.7213 | LR: 4.66e-04 | 51429.9ms/step | 1.2 samples/s
286
+ Step 1770/10005 | Loss: 3.7331 | LR: 4.66e-04 | 2666.8ms/step | 24.0 samples/s
287
+ Step 1780/10005 | Loss: 3.7389 | LR: 4.65e-04 | 2665.1ms/step | 24.0 samples/s
288
+ Step 1790/10005 | Loss: 3.7358 | LR: 4.65e-04 | 2665.1ms/step | 24.0 samples/s
289
+ Step 1800/10005 | Loss: 3.7150 | LR: 4.65e-04 | 2664.3ms/step | 24.0 samples/s
290
+ Step 1810/10005 | Loss: 3.7146 | LR: 4.64e-04 | 2663.6ms/step | 24.0 samples/s
291
+ Step 1820/10005 | Loss: 3.7027 | LR: 4.64e-04 | 2662.6ms/step | 24.0 samples/s
292
+ Step 1830/10005 | Loss: 3.7183 | LR: 4.63e-04 | 2663.8ms/step | 24.0 samples/s
293
+ Step 1840/10005 | Loss: 3.7229 | LR: 4.63e-04 | 2662.7ms/step | 24.0 samples/s
294
+ Step 1850/10005 | Loss: 3.7111 | LR: 4.62e-04 | 2660.0ms/step | 24.1 samples/s
295
+ Step 1860/10005 | Loss: 3.7052 | LR: 4.62e-04 | 2664.1ms/step | 24.0 samples/s
296
+ Step 1870/10005 | Loss: 3.6911 | LR: 4.62e-04 | 2659.6ms/step | 24.1 samples/s
297
+ Step 1880/10005 | Loss: 3.7114 | LR: 4.61e-04 | 2661.8ms/step | 24.0 samples/s
298
+ Step 1890/10005 | Loss: 3.6985 | LR: 4.61e-04 | 2660.9ms/step | 24.1 samples/s
299
+ Step 1900/10005 | Loss: 3.7025 | LR: 4.60e-04 | 2661.1ms/step | 24.1 samples/s
300
+ Step 1910/10005 | Loss: 3.7191 | LR: 4.60e-04 | 2661.4ms/step | 24.0 samples/s
301
+ Step 1920/10005 | Loss: 3.6931 | LR: 4.59e-04 | 2662.6ms/step | 24.0 samples/s
302
+ Step 1930/10005 | Loss: 3.6913 | LR: 4.59e-04 | 2662.6ms/step | 24.0 samples/s
303
+ Step 1940/10005 | Loss: 3.6991 | LR: 4.59e-04 | 2665.2ms/step | 24.0 samples/s
304
+ Step 1950/10005 | Loss: 3.6824 | LR: 4.58e-04 | 2664.1ms/step | 24.0 samples/s
305
+ Step 1960/10005 | Loss: 3.6785 | LR: 4.58e-04 | 2664.0ms/step | 24.0 samples/s
306
+ Step 1970/10005 | Loss: 3.6668 | LR: 4.57e-04 | 2663.4ms/step | 24.0 samples/s
307
+ Step 1980/10005 | Loss: 3.6743 | LR: 4.57e-04 | 2662.8ms/step | 24.0 samples/s
308
+ Step 1990/10005 | Loss: 3.6675 | LR: 4.56e-04 | 2663.2ms/step | 24.0 samples/s
309
+ Step 2000/10005 | Loss: 3.6640 | LR: 4.56e-04 | 2664.6ms/step | 24.0 samples/s
310
+ Validation at step 2000: Loss = 3.6766, Perplexity = 39.51
311
+ Step 2010/10005 | Loss: 3.6533 | LR: 4.56e-04 | 51398.4ms/step | 1.2 samples/s
312
+ Step 2020/10005 | Loss: 3.6735 | LR: 4.55e-04 | 2662.0ms/step | 24.0 samples/s
313
+ Step 2030/10005 | Loss: 3.6641 | LR: 4.55e-04 | 2660.0ms/step | 24.1 samples/s
314
+ Step 2040/10005 | Loss: 3.6645 | LR: 4.54e-04 | 2661.2ms/step | 24.0 samples/s
315
+ Step 2050/10005 | Loss: 3.6912 | LR: 4.54e-04 | 2664.4ms/step | 24.0 samples/s
316
+ Step 2060/10005 | Loss: 3.6588 | LR: 4.53e-04 | 2665.2ms/step | 24.0 samples/s
317
+ Step 2070/10005 | Loss: 3.6689 | LR: 4.53e-04 | 2665.0ms/step | 24.0 samples/s
318
+ Step 2080/10005 | Loss: 3.6537 | LR: 4.52e-04 | 2662.2ms/step | 24.0 samples/s
319
+ Step 2090/10005 | Loss: 3.6348 | LR: 4.52e-04 | 2665.3ms/step | 24.0 samples/s
320
+ Step 2100/10005 | Loss: 3.6609 | LR: 4.51e-04 | 2664.2ms/step | 24.0 samples/s
321
+ Step 2110/10005 | Loss: 3.6499 | LR: 4.51e-04 | 2665.0ms/step | 24.0 samples/s
322
+ Step 2120/10005 | Loss: 3.6646 | LR: 4.50e-04 | 2662.2ms/step | 24.0 samples/s
323
+ Step 2130/10005 | Loss: 3.6313 | LR: 4.50e-04 | 2661.6ms/step | 24.0 samples/s
324
+ Step 2140/10005 | Loss: 3.6630 | LR: 4.49e-04 | 2660.0ms/step | 24.1 samples/s
325
+ Step 2150/10005 | Loss: 3.6742 | LR: 4.49e-04 | 2664.0ms/step | 24.0 samples/s
326
+ Step 2160/10005 | Loss: 3.6416 | LR: 4.49e-04 | 2663.4ms/step | 24.0 samples/s
327
+ Step 2170/10005 | Loss: 3.6283 | LR: 4.48e-04 | 2664.4ms/step | 24.0 samples/s
328
+ Step 2180/10005 | Loss: 3.6184 | LR: 4.48e-04 | 2665.4ms/step | 24.0 samples/s
329
+ Step 2190/10005 | Loss: 3.6224 | LR: 4.47e-04 | 2660.4ms/step | 24.1 samples/s
330
+ Step 2200/10005 | Loss: 3.6361 | LR: 4.47e-04 | 2663.0ms/step | 24.0 samples/s
331
+ Step 2210/10005 | Loss: 3.6381 | LR: 4.46e-04 | 2661.4ms/step | 24.0 samples/s
332
+ Step 2220/10005 | Loss: 3.6436 | LR: 4.46e-04 | 2660.1ms/step | 24.1 samples/s
333
+ Step 2230/10005 | Loss: 3.6527 | LR: 4.45e-04 | 2667.2ms/step | 24.0 samples/s
334
+ Step 2240/10005 | Loss: 3.6479 | LR: 4.45e-04 | 2660.8ms/step | 24.1 samples/s
335
+ Step 2250/10005 | Loss: 3.6112 | LR: 4.44e-04 | 2662.5ms/step | 24.0 samples/s
336
+ Validation at step 2250: Loss = 3.6225, Perplexity = 37.43
337
+ Step 2260/10005 | Loss: 3.6295 | LR: 4.44e-04 | 51410.7ms/step | 1.2 samples/s
338
+ Step 2270/10005 | Loss: 3.6021 | LR: 4.43e-04 | 2662.7ms/step | 24.0 samples/s
339
+ Step 2280/10005 | Loss: 3.6127 | LR: 4.43e-04 | 2664.1ms/step | 24.0 samples/s
340
+ Step 2290/10005 | Loss: 3.6335 | LR: 4.42e-04 | 2664.8ms/step | 24.0 samples/s
341
+ Step 2300/10005 | Loss: 3.6134 | LR: 4.42e-04 | 2664.6ms/step | 24.0 samples/s
342
+ Step 2310/10005 | Loss: 3.6236 | LR: 4.41e-04 | 2667.4ms/step | 24.0 samples/s
343
+ Step 2320/10005 | Loss: 3.6047 | LR: 4.41e-04 | 2669.1ms/step | 24.0 samples/s
344
+ Step 2330/10005 | Loss: 3.5930 | LR: 4.40e-04 | 2666.4ms/step | 24.0 samples/s
345
+ Step 2340/10005 | Loss: 3.6012 | LR: 4.40e-04 | 2663.0ms/step | 24.0 samples/s
346
+ Step 2350/10005 | Loss: 3.6072 | LR: 4.39e-04 | 2664.5ms/step | 24.0 samples/s
347
+ Step 2360/10005 | Loss: 3.6260 | LR: 4.38e-04 | 2665.1ms/step | 24.0 samples/s
348
+ Step 2370/10005 | Loss: 3.5977 | LR: 4.38e-04 | 2664.3ms/step | 24.0 samples/s
349
+ Step 2380/10005 | Loss: 3.6183 | LR: 4.37e-04 | 2662.9ms/step | 24.0 samples/s
350
+ Step 2390/10005 | Loss: 3.5863 | LR: 4.37e-04 | 2664.6ms/step | 24.0 samples/s
351
+ Step 2400/10005 | Loss: 3.5840 | LR: 4.36e-04 | 2663.4ms/step | 24.0 samples/s
352
+ Step 2410/10005 | Loss: 3.5828 | LR: 4.36e-04 | 2662.8ms/step | 24.0 samples/s
353
+ Step 2420/10005 | Loss: 3.5914 | LR: 4.35e-04 | 2662.5ms/step | 24.0 samples/s
354
+ Step 2430/10005 | Loss: 3.5825 | LR: 4.35e-04 | 2661.2ms/step | 24.0 samples/s
355
+ Step 2440/10005 | Loss: 3.5847 | LR: 4.34e-04 | 2659.0ms/step | 24.1 samples/s
356
+ Step 2450/10005 | Loss: 3.5516 | LR: 4.34e-04 | 2662.0ms/step | 24.0 samples/s
357
+ Step 2460/10005 | Loss: 3.5753 | LR: 4.33e-04 | 2661.1ms/step | 24.1 samples/s
358
+ Step 2470/10005 | Loss: 3.6349 | LR: 4.33e-04 | 2661.1ms/step | 24.0 samples/s
359
+ Step 2480/10005 | Loss: 3.5752 | LR: 4.32e-04 | 2661.2ms/step | 24.0 samples/s
360
+ Step 2490/10005 | Loss: 3.5842 | LR: 4.32e-04 | 2661.6ms/step | 24.0 samples/s
361
+ Step 2500/10005 | Loss: 3.5774 | LR: 4.31e-04 | 2664.0ms/step | 24.0 samples/s
362
+ Validation at step 2500: Loss = 3.5793, Perplexity = 35.85
363
+ Saved checkpoint to checkpoints/run_20260430_153357/checkpoint-2500
364
+ Step 2510/10005 | Loss: 3.5876 | LR: 4.30e-04 | 51895.3ms/step | 1.2 samples/s
365
+ Step 2520/10005 | Loss: 3.5677 | LR: 4.30e-04 | 2661.9ms/step | 24.0 samples/s
366
+ Step 2530/10005 | Loss: 3.5929 | LR: 4.29e-04 | 2662.2ms/step | 24.0 samples/s
367
+ Step 2540/10005 | Loss: 3.5598 | LR: 4.29e-04 | 2661.5ms/step | 24.0 samples/s
368
+ Step 2550/10005 | Loss: 3.5745 | LR: 4.28e-04 | 2660.2ms/step | 24.1 samples/s
369
+ Step 2560/10005 | Loss: 3.5692 | LR: 4.28e-04 | 2661.5ms/step | 24.0 samples/s
370
+ Step 2570/10005 | Loss: 3.5736 | LR: 4.27e-04 | 2662.8ms/step | 24.0 samples/s
371
+ Step 2580/10005 | Loss: 3.5860 | LR: 4.27e-04 | 2662.9ms/step | 24.0 samples/s
372
+ Step 2590/10005 | Loss: 3.5657 | LR: 4.26e-04 | 2663.7ms/step | 24.0 samples/s
373
+ Step 2600/10005 | Loss: 3.5627 | LR: 4.25e-04 | 2661.0ms/step | 24.1 samples/s
374
+ Step 2610/10005 | Loss: 3.5421 | LR: 4.25e-04 | 2665.5ms/step | 24.0 samples/s
375
+ Step 2620/10005 | Loss: 3.5378 | LR: 4.24e-04 | 2662.3ms/step | 24.0 samples/s
376
+ Step 2630/10005 | Loss: 3.5739 | LR: 4.24e-04 | 2662.9ms/step | 24.0 samples/s
377
+ Step 2640/10005 | Loss: 3.5352 | LR: 4.23e-04 | 2663.3ms/step | 24.0 samples/s
378
+ Step 2650/10005 | Loss: 3.5437 | LR: 4.23e-04 | 2662.3ms/step | 24.0 samples/s
379
+ Step 2660/10005 | Loss: 3.5579 | LR: 4.22e-04 | 2664.1ms/step | 24.0 samples/s
380
+ Step 2670/10005 | Loss: 3.5532 | LR: 4.21e-04 | 2663.7ms/step | 24.0 samples/s
381
+ Step 2680/10005 | Loss: 3.5470 | LR: 4.21e-04 | 2661.8ms/step | 24.0 samples/s
382
+ Step 2690/10005 | Loss: 3.5507 | LR: 4.20e-04 | 2664.6ms/step | 24.0 samples/s
383
+ Step 2700/10005 | Loss: 3.5683 | LR: 4.20e-04 | 2665.4ms/step | 24.0 samples/s
384
+ Step 2710/10005 | Loss: 3.5616 | LR: 4.19e-04 | 2663.5ms/step | 24.0 samples/s
385
+ Step 2720/10005 | Loss: 3.5387 | LR: 4.19e-04 | 2664.2ms/step | 24.0 samples/s
386
+ Step 2730/10005 | Loss: 3.5183 | LR: 4.18e-04 | 2664.3ms/step | 24.0 samples/s
387
+ Step 2740/10005 | Loss: 3.5247 | LR: 4.17e-04 | 2662.2ms/step | 24.0 samples/s
388
+ Step 2750/10005 | Loss: 3.5397 | LR: 4.17e-04 | 2659.7ms/step | 24.1 samples/s
389
+ Validation at step 2750: Loss = 3.5397, Perplexity = 34.46
390
+ Step 2760/10005 | Loss: 3.5514 | LR: 4.16e-04 | 51427.9ms/step | 1.2 samples/s
391
+ Step 2770/10005 | Loss: 3.5514 | LR: 4.16e-04 | 2662.4ms/step | 24.0 samples/s
392
+ Step 2780/10005 | Loss: 3.5307 | LR: 4.15e-04 | 2663.9ms/step | 24.0 samples/s
393
+ Step 2790/10005 | Loss: 3.5258 | LR: 4.14e-04 | 2664.1ms/step | 24.0 samples/s
394
+ Step 2800/10005 | Loss: 3.4972 | LR: 4.14e-04 | 2664.1ms/step | 24.0 samples/s
395
+ Step 2810/10005 | Loss: 3.5242 | LR: 4.13e-04 | 2667.3ms/step | 24.0 samples/s
396
+ Step 2820/10005 | Loss: 3.5292 | LR: 4.13e-04 | 2661.7ms/step | 24.0 samples/s
397
+ Step 2830/10005 | Loss: 3.5179 | LR: 4.12e-04 | 2662.5ms/step | 24.0 samples/s
398
+ Step 2840/10005 | Loss: 3.5334 | LR: 4.11e-04 | 2664.1ms/step | 24.0 samples/s
399
+ Step 2850/10005 | Loss: 3.4889 | LR: 4.11e-04 | 2665.6ms/step | 24.0 samples/s
400
+ Step 2860/10005 | Loss: 3.5179 | LR: 4.10e-04 | 2662.3ms/step | 24.0 samples/s
401
+ Step 2870/10005 | Loss: 3.5098 | LR: 4.10e-04 | 2662.0ms/step | 24.0 samples/s
402
+ Step 2880/10005 | Loss: 3.5155 | LR: 4.09e-04 | 2659.6ms/step | 24.1 samples/s
403
+ Step 2890/10005 | Loss: 3.5112 | LR: 4.08e-04 | 2665.6ms/step | 24.0 samples/s
404
+ Step 2900/10005 | Loss: 3.5145 | LR: 4.08e-04 | 2661.9ms/step | 24.0 samples/s
405
+ Step 2910/10005 | Loss: 3.5058 | LR: 4.07e-04 | 2662.8ms/step | 24.0 samples/s
406
+ Step 2920/10005 | Loss: 3.5280 | LR: 4.06e-04 | 2662.6ms/step | 24.0 samples/s
407
+ Step 2930/10005 | Loss: 3.4985 | LR: 4.06e-04 | 2659.9ms/step | 24.1 samples/s
408
+ Step 2940/10005 | Loss: 3.5129 | LR: 4.05e-04 | 2661.6ms/step | 24.0 samples/s
409
+ Step 2950/10005 | Loss: 3.5292 | LR: 4.05e-04 | 2663.5ms/step | 24.0 samples/s
410
+ Step 2960/10005 | Loss: 3.4884 | LR: 4.04e-04 | 2662.4ms/step | 24.0 samples/s
411
+ Step 2970/10005 | Loss: 3.5055 | LR: 4.03e-04 | 2666.3ms/step | 24.0 samples/s
412
+ Step 2980/10005 | Loss: 3.4804 | LR: 4.03e-04 | 2663.4ms/step | 24.0 samples/s
413
+ Step 2990/10005 | Loss: 3.4840 | LR: 4.02e-04 | 2663.4ms/step | 24.0 samples/s
414
+ Step 3000/10005 | Loss: 3.4983 | LR: 4.01e-04 | 2663.2ms/step | 24.0 samples/s
415
+ Validation at step 3000: Loss = 3.5053, Perplexity = 33.29
416
+ Step 3010/10005 | Loss: 3.4955 | LR: 4.01e-04 | 51404.7ms/step | 1.2 samples/s
417
+ Step 3020/10005 | Loss: 3.4946 | LR: 4.00e-04 | 2663.0ms/step | 24.0 samples/s
418
+ Step 3030/10005 | Loss: 3.4903 | LR: 4.00e-04 | 2664.2ms/step | 24.0 samples/s
419
+ Step 3040/10005 | Loss: 3.4907 | LR: 3.99e-04 | 2663.7ms/step | 24.0 samples/s
420
+ Step 3050/10005 | Loss: 3.5003 | LR: 3.98e-04 | 2663.7ms/step | 24.0 samples/s
421
+ Step 3060/10005 | Loss: 3.4987 | LR: 3.98e-04 | 2662.2ms/step | 24.0 samples/s
422
+ Step 3070/10005 | Loss: 3.4935 | LR: 3.97e-04 | 2664.8ms/step | 24.0 samples/s
423
+ Step 3080/10005 | Loss: 3.4824 | LR: 3.96e-04 | 2660.8ms/step | 24.1 samples/s
424
+ Step 3090/10005 | Loss: 3.4836 | LR: 3.96e-04 | 2661.3ms/step | 24.0 samples/s
425
+ Step 3100/10005 | Loss: 3.4852 | LR: 3.95e-04 | 2664.0ms/step | 24.0 samples/s
426
+ Step 3110/10005 | Loss: 3.5003 | LR: 3.94e-04 | 2662.9ms/step | 24.0 samples/s
427
+ Step 3120/10005 | Loss: 3.5107 | LR: 3.94e-04 | 2658.4ms/step | 24.1 samples/s
428
+ Step 3130/10005 | Loss: 3.4891 | LR: 3.93e-04 | 2659.7ms/step | 24.1 samples/s
429
+ Step 3140/10005 | Loss: 3.4736 | LR: 3.93e-04 | 2663.1ms/step | 24.0 samples/s
430
+ Step 3150/10005 | Loss: 3.4848 | LR: 3.92e-04 | 2661.5ms/step | 24.0 samples/s
431
+ Step 3160/10005 | Loss: 3.4643 | LR: 3.91e-04 | 2663.2ms/step | 24.0 samples/s
432
+ Step 3170/10005 | Loss: 3.4711 | LR: 3.91e-04 | 2659.2ms/step | 24.1 samples/s
433
+ Step 3180/10005 | Loss: 3.4960 | LR: 3.90e-04 | 2658.3ms/step | 24.1 samples/s
434
+ Step 3190/10005 | Loss: 3.4864 | LR: 3.89e-04 | 2661.4ms/step | 24.0 samples/s
435
+ Step 3200/10005 | Loss: 3.4776 | LR: 3.89e-04 | 2664.0ms/step | 24.0 samples/s
436
+ Step 3210/10005 | Loss: 3.4792 | LR: 3.88e-04 | 2663.9ms/step | 24.0 samples/s
437
+ Step 3220/10005 | Loss: 3.4737 | LR: 3.87e-04 | 2661.9ms/step | 24.0 samples/s
438
+ Step 3230/10005 | Loss: 3.4655 | LR: 3.87e-04 | 2661.2ms/step | 24.0 samples/s
439
+ Step 3240/10005 | Loss: 3.4828 | LR: 3.86e-04 | 2662.3ms/step | 24.0 samples/s
440
+ Step 3250/10005 | Loss: 3.4717 | LR: 3.85e-04 | 2663.4ms/step | 24.0 samples/s
441
+ Validation at step 3250: Loss = 3.4748, Perplexity = 32.29
442
+ Step 3260/10005 | Loss: 3.4814 | LR: 3.85e-04 | 51429.4ms/step | 1.2 samples/s
443
+ Step 3270/10005 | Loss: 3.4711 | LR: 3.84e-04 | 2662.9ms/step | 24.0 samples/s
444
+ Step 3280/10005 | Loss: 3.4713 | LR: 3.83e-04 | 2663.7ms/step | 24.0 samples/s
445
+ Step 3290/10005 | Loss: 3.4687 | LR: 3.83e-04 | 2664.4ms/step | 24.0 samples/s
446
+ Step 3300/10005 | Loss: 3.4824 | LR: 3.82e-04 | 2660.8ms/step | 24.1 samples/s
447
+ Step 3310/10005 | Loss: 3.4582 | LR: 3.81e-04 | 2665.0ms/step | 24.0 samples/s
448
+ Step 3320/10005 | Loss: 3.4743 | LR: 3.81e-04 | 2664.4ms/step | 24.0 samples/s
449
+ Step 3330/10005 | Loss: 3.4607 | LR: 3.80e-04 | 2663.7ms/step | 24.0 samples/s
450
+ Step 3340/10005 | Loss: 3.4540 | LR: 3.79e-04 | 2664.5ms/step | 24.0 samples/s
451
+ Step 3350/10005 | Loss: 3.4904 | LR: 3.79e-04 | 2662.3ms/step | 24.0 samples/s
452
+ Step 3360/10005 | Loss: 3.4483 | LR: 3.78e-04 | 2663.3ms/step | 24.0 samples/s
453
+ Step 3370/10005 | Loss: 3.4458 | LR: 3.77e-04 | 2664.8ms/step | 24.0 samples/s
454
+ Step 3380/10005 | Loss: 3.4523 | LR: 3.76e-04 | 2663.3ms/step | 24.0 samples/s
455
+ Step 3390/10005 | Loss: 3.4396 | LR: 3.76e-04 | 2662.1ms/step | 24.0 samples/s
456
+ Step 3400/10005 | Loss: 3.4714 | LR: 3.75e-04 | 2663.7ms/step | 24.0 samples/s
457
+ Step 3410/10005 | Loss: 3.4637 | LR: 3.74e-04 | 2663.5ms/step | 24.0 samples/s
458
+ Step 3420/10005 | Loss: 3.4585 | LR: 3.74e-04 | 2662.2ms/step | 24.0 samples/s
459
+ Step 3430/10005 | Loss: 3.4539 | LR: 3.73e-04 | 2663.1ms/step | 24.0 samples/s
460
+ Step 3440/10005 | Loss: 3.4444 | LR: 3.72e-04 | 2660.5ms/step | 24.1 samples/s
461
+ Step 3450/10005 | Loss: 3.4635 | LR: 3.72e-04 | 2662.7ms/step | 24.0 samples/s
462
+ Step 3460/10005 | Loss: 3.4368 | LR: 3.71e-04 | 2663.6ms/step | 24.0 samples/s
463
+ Step 3470/10005 | Loss: 3.4488 | LR: 3.70e-04 | 2662.0ms/step | 24.0 samples/s
464
+ Step 3480/10005 | Loss: 3.4341 | LR: 3.70e-04 | 2663.6ms/step | 24.0 samples/s
465
+ Step 3490/10005 | Loss: 3.4411 | LR: 3.69e-04 | 2662.9ms/step | 24.0 samples/s
466
+ Step 3500/10005 | Loss: 3.4465 | LR: 3.68e-04 | 2661.7ms/step | 24.0 samples/s
467
+ Validation at step 3500: Loss = 3.4472, Perplexity = 31.41
468
+ Step 3510/10005 | Loss: 3.4362 | LR: 3.67e-04 | 51389.1ms/step | 1.2 samples/s
469
+ Step 3520/10005 | Loss: 3.4300 | LR: 3.67e-04 | 2665.9ms/step | 24.0 samples/s
470
+ Step 3530/10005 | Loss: 3.4429 | LR: 3.66e-04 | 2663.3ms/step | 24.0 samples/s
471
+ Step 3540/10005 | Loss: 3.4517 | LR: 3.65e-04 | 2664.1ms/step | 24.0 samples/s
472
+ Step 3550/10005 | Loss: 3.4373 | LR: 3.65e-04 | 2662.6ms/step | 24.0 samples/s
473
+ Step 3560/10005 | Loss: 3.4324 | LR: 3.64e-04 | 2662.2ms/step | 24.0 samples/s
474
+ Step 3570/10005 | Loss: 3.4279 | LR: 3.63e-04 | 2660.6ms/step | 24.1 samples/s
475
+ Step 3580/10005 | Loss: 3.4454 | LR: 3.63e-04 | 2661.4ms/step | 24.0 samples/s
476
+ Step 3590/10005 | Loss: 3.4444 | LR: 3.62e-04 | 2663.3ms/step | 24.0 samples/s
477
+ Step 3600/10005 | Loss: 3.4212 | LR: 3.61e-04 | 2662.6ms/step | 24.0 samples/s
478
+ Step 3610/10005 | Loss: 3.4392 | LR: 3.60e-04 | 2664.3ms/step | 24.0 samples/s
479
+ Step 3620/10005 | Loss: 3.4492 | LR: 3.60e-04 | 2665.0ms/step | 24.0 samples/s
480
+ Step 3630/10005 | Loss: 3.4393 | LR: 3.59e-04 | 2660.6ms/step | 24.1 samples/s
481
+ Step 3640/10005 | Loss: 3.4469 | LR: 3.58e-04 | 2663.6ms/step | 24.0 samples/s
482
+ Step 3650/10005 | Loss: 3.4200 | LR: 3.58e-04 | 2662.3ms/step | 24.0 samples/s
483
+ Step 3660/10005 | Loss: 3.4287 | LR: 3.57e-04 | 2664.0ms/step | 24.0 samples/s
484
+ Step 3670/10005 | Loss: 3.4242 | LR: 3.56e-04 | 2662.8ms/step | 24.0 samples/s
485
+ Step 3680/10005 | Loss: 3.4294 | LR: 3.55e-04 | 2664.3ms/step | 24.0 samples/s
486
+ Step 3690/10005 | Loss: 3.4275 | LR: 3.55e-04 | 2660.7ms/step | 24.1 samples/s
487
+ Step 3700/10005 | Loss: 3.4277 | LR: 3.54e-04 | 2663.6ms/step | 24.0 samples/s
488
+ Step 3710/10005 | Loss: 3.4152 | LR: 3.53e-04 | 2662.5ms/step | 24.0 samples/s
489
+ Step 3720/10005 | Loss: 3.4029 | LR: 3.53e-04 | 2665.2ms/step | 24.0 samples/s
490
+ Step 3730/10005 | Loss: 3.4311 | LR: 3.52e-04 | 2662.7ms/step | 24.0 samples/s
491
+ Step 3740/10005 | Loss: 3.4120 | LR: 3.51e-04 | 2664.2ms/step | 24.0 samples/s
492
+ Step 3750/10005 | Loss: 3.4023 | LR: 3.50e-04 | 2661.0ms/step | 24.1 samples/s
493
+ Validation at step 3750: Loss = 3.4220, Perplexity = 30.63
494
+ Step 3760/10005 | Loss: 3.4090 | LR: 3.50e-04 | 51417.0ms/step | 1.2 samples/s
495
+ Step 3770/10005 | Loss: 3.4389 | LR: 3.49e-04 | 2665.5ms/step | 24.0 samples/s
496
+ Step 3780/10005 | Loss: 3.4401 | LR: 3.48e-04 | 2665.2ms/step | 24.0 samples/s
497
+ Step 3790/10005 | Loss: 3.4129 | LR: 3.47e-04 | 2662.7ms/step | 24.0 samples/s
498
+ Step 3800/10005 | Loss: 3.4040 | LR: 3.47e-04 | 2665.4ms/step | 24.0 samples/s
499
+ Step 3810/10005 | Loss: 3.4013 | LR: 3.46e-04 | 2662.6ms/step | 24.0 samples/s
500
+ Step 3820/10005 | Loss: 3.4319 | LR: 3.45e-04 | 2664.7ms/step | 24.0 samples/s
501
+ Step 3830/10005 | Loss: 3.4172 | LR: 3.45e-04 | 2663.3ms/step | 24.0 samples/s
502
+ Step 3840/10005 | Loss: 3.4021 | LR: 3.44e-04 | 2662.7ms/step | 24.0 samples/s
503
+ Step 3850/10005 | Loss: 3.4220 | LR: 3.43e-04 | 2663.6ms/step | 24.0 samples/s
504
+ Step 3860/10005 | Loss: 3.3990 | LR: 3.42e-04 | 2661.2ms/step | 24.0 samples/s
505
+ Step 3870/10005 | Loss: 3.3766 | LR: 3.42e-04 | 2660.1ms/step | 24.1 samples/s
506
+ Step 3880/10005 | Loss: 3.4187 | LR: 3.41e-04 | 2661.7ms/step | 24.0 samples/s
507
+ Step 3890/10005 | Loss: 3.4017 | LR: 3.40e-04 | 2663.7ms/step | 24.0 samples/s
508
+ Step 3900/10005 | Loss: 3.3869 | LR: 3.39e-04 | 2663.0ms/step | 24.0 samples/s
509
+ Step 3910/10005 | Loss: 3.4274 | LR: 3.39e-04 | 2659.9ms/step | 24.1 samples/s
510
+ Step 3920/10005 | Loss: 3.3942 | LR: 3.38e-04 | 2661.8ms/step | 24.0 samples/s
511
+ Step 3930/10005 | Loss: 3.4177 | LR: 3.37e-04 | 2663.1ms/step | 24.0 samples/s
512
+ Step 3940/10005 | Loss: 3.4002 | LR: 3.36e-04 | 2663.9ms/step | 24.0 samples/s
513
+ Step 3950/10005 | Loss: 3.3877 | LR: 3.36e-04 | 2664.3ms/step | 24.0 samples/s
514
+ Step 3960/10005 | Loss: 3.4087 | LR: 3.35e-04 | 2664.6ms/step | 24.0 samples/s
515
+ Step 3970/10005 | Loss: 3.3933 | LR: 3.34e-04 | 2663.2ms/step | 24.0 samples/s
516
+ Step 3980/10005 | Loss: 3.4147 | LR: 3.33e-04 | 2661.7ms/step | 24.0 samples/s
517
+ Step 3990/10005 | Loss: 3.3950 | LR: 3.33e-04 | 2664.2ms/step | 24.0 samples/s
518
+ Step 4000/10005 | Loss: 3.3926 | LR: 3.32e-04 | 2661.5ms/step | 24.0 samples/s
519
+ Validation at step 4000: Loss = 3.4001, Perplexity = 29.97
520
+ Step 4010/10005 | Loss: 3.4202 | LR: 3.31e-04 | 51419.6ms/step | 1.2 samples/s
521
+ Step 4020/10005 | Loss: 3.4127 | LR: 3.30e-04 | 2662.0ms/step | 24.0 samples/s
522
+ Step 4030/10005 | Loss: 3.4102 | LR: 3.30e-04 | 2662.0ms/step | 24.0 samples/s
523
+ Step 4040/10005 | Loss: 3.3901 | LR: 3.29e-04 | 2662.6ms/step | 24.0 samples/s
524
+ Step 4050/10005 | Loss: 3.4062 | LR: 3.28e-04 | 2665.6ms/step | 24.0 samples/s
525
+ Step 4060/10005 | Loss: 3.3952 | LR: 3.27e-04 | 2661.1ms/step | 24.1 samples/s
526
+ Step 4070/10005 | Loss: 3.3614 | LR: 3.27e-04 | 2660.3ms/step | 24.1 samples/s
527
+ Step 4080/10005 | Loss: 3.3864 | LR: 3.26e-04 | 2663.1ms/step | 24.0 samples/s
528
+ Step 4090/10005 | Loss: 3.4032 | LR: 3.25e-04 | 2664.7ms/step | 24.0 samples/s
529
+ Step 4100/10005 | Loss: 3.4086 | LR: 3.24e-04 | 2661.9ms/step | 24.0 samples/s
530
+ Step 4110/10005 | Loss: 3.3727 | LR: 3.24e-04 | 2662.2ms/step | 24.0 samples/s
531
+ Step 4120/10005 | Loss: 3.3783 | LR: 3.23e-04 | 2659.4ms/step | 24.1 samples/s
532
+ Step 4130/10005 | Loss: 3.3917 | LR: 3.22e-04 | 2661.4ms/step | 24.0 samples/s
533
+ Step 4140/10005 | Loss: 3.4169 | LR: 3.21e-04 | 2663.7ms/step | 24.0 samples/s
534
+ Step 4150/10005 | Loss: 3.3788 | LR: 3.21e-04 | 2660.2ms/step | 24.1 samples/s
535
+ Step 4160/10005 | Loss: 3.3880 | LR: 3.20e-04 | 2664.8ms/step | 24.0 samples/s
536
+ Step 4170/10005 | Loss: 3.3784 | LR: 3.19e-04 | 2663.5ms/step | 24.0 samples/s
537
+ Step 4180/10005 | Loss: 3.3739 | LR: 3.18e-04 | 2660.2ms/step | 24.1 samples/s
538
+ Step 4190/10005 | Loss: 3.3731 | LR: 3.18e-04 | 2662.3ms/step | 24.0 samples/s
539
+ Step 4200/10005 | Loss: 3.3493 | LR: 3.17e-04 | 2663.4ms/step | 24.0 samples/s
540
+ Step 4210/10005 | Loss: 3.3930 | LR: 3.16e-04 | 2663.1ms/step | 24.0 samples/s
541
+ Step 4220/10005 | Loss: 3.3896 | LR: 3.15e-04 | 2663.0ms/step | 24.0 samples/s
542
+ Step 4230/10005 | Loss: 3.3780 | LR: 3.14e-04 | 2662.6ms/step | 24.0 samples/s
543
+ Step 4240/10005 | Loss: 3.3764 | LR: 3.14e-04 | 2659.3ms/step | 24.1 samples/s
544
+ Step 4250/10005 | Loss: 3.3844 | LR: 3.13e-04 | 2661.8ms/step | 24.0 samples/s
545
+ Validation at step 4250: Loss = 3.3783, Perplexity = 29.32
546
+ Step 4260/10005 | Loss: 3.3555 | LR: 3.12e-04 | 51402.7ms/step | 1.2 samples/s
547
+ Step 4270/10005 | Loss: 3.3885 | LR: 3.11e-04 | 2667.1ms/step | 24.0 samples/s
548
+ Step 4280/10005 | Loss: 3.3713 | LR: 3.11e-04 | 2661.6ms/step | 24.0 samples/s
549
+ Step 4290/10005 | Loss: 3.3524 | LR: 3.10e-04 | 2661.3ms/step | 24.0 samples/s
550
+ Step 4300/10005 | Loss: 3.3879 | LR: 3.09e-04 | 2659.9ms/step | 24.1 samples/s
551
+ Step 4310/10005 | Loss: 3.3709 | LR: 3.08e-04 | 2663.0ms/step | 24.0 samples/s
552
+ Step 4320/10005 | Loss: 3.3639 | LR: 3.08e-04 | 2661.5ms/step | 24.0 samples/s
553
+ Step 4330/10005 | Loss: 3.3675 | LR: 3.07e-04 | 2664.3ms/step | 24.0 samples/s
554
+ Step 4340/10005 | Loss: 3.3906 | LR: 3.06e-04 | 2661.8ms/step | 24.0 samples/s
555
+ Step 4350/10005 | Loss: 3.3783 | LR: 3.05e-04 | 2661.6ms/step | 24.0 samples/s
556
+ Step 4360/10005 | Loss: 3.3818 | LR: 3.04e-04 | 2661.8ms/step | 24.0 samples/s
557
+ Step 4370/10005 | Loss: 3.3713 | LR: 3.04e-04 | 2662.8ms/step | 24.0 samples/s
558
+ Step 4380/10005 | Loss: 3.3919 | LR: 3.03e-04 | 2661.5ms/step | 24.0 samples/s
559
+ Step 4390/10005 | Loss: 3.3373 | LR: 3.02e-04 | 2660.1ms/step | 24.1 samples/s
560
+ Step 4400/10005 | Loss: 3.3351 | LR: 3.01e-04 | 2660.5ms/step | 24.1 samples/s
561
+ Step 4410/10005 | Loss: 3.3685 | LR: 3.01e-04 | 2661.5ms/step | 24.0 samples/s
562
+ Step 4420/10005 | Loss: 3.3725 | LR: 3.00e-04 | 2656.6ms/step | 24.1 samples/s
563
+ Step 4430/10005 | Loss: 3.3592 | LR: 2.99e-04 | 2658.6ms/step | 24.1 samples/s
564
+ Step 4440/10005 | Loss: 3.3633 | LR: 2.98e-04 | 2658.8ms/step | 24.1 samples/s
565
+ Step 4450/10005 | Loss: 3.3535 | LR: 2.97e-04 | 2657.5ms/step | 24.1 samples/s
566
+ Step 4460/10005 | Loss: 3.3661 | LR: 2.97e-04 | 2656.8ms/step | 24.1 samples/s
567
+ Step 4470/10005 | Loss: 3.3706 | LR: 2.96e-04 | 2657.1ms/step | 24.1 samples/s
568
+ Step 4480/10005 | Loss: 3.3507 | LR: 2.95e-04 | 2657.9ms/step | 24.1 samples/s
569
+ Step 4490/10005 | Loss: 3.3415 | LR: 2.94e-04 | 2659.3ms/step | 24.1 samples/s
570
+ Step 4500/10005 | Loss: 3.3570 | LR: 2.94e-04 | 2659.8ms/step | 24.1 samples/s
571
+ Validation at step 4500: Loss = 3.3584, Perplexity = 28.74
572
+ Step 4510/10005 | Loss: 3.3472 | LR: 2.93e-04 | 51375.2ms/step | 1.2 samples/s
573
+ Step 4520/10005 | Loss: 3.3769 | LR: 2.92e-04 | 2660.4ms/step | 24.1 samples/s
574
+ Step 4530/10005 | Loss: 3.3521 | LR: 2.91e-04 | 2660.3ms/step | 24.1 samples/s
575
+ Step 4540/10005 | Loss: 3.3476 | LR: 2.90e-04 | 2660.2ms/step | 24.1 samples/s
576
+ Step 4550/10005 | Loss: 3.3461 | LR: 2.90e-04 | 2661.0ms/step | 24.1 samples/s
577
+ Step 4560/10005 | Loss: 3.3565 | LR: 2.89e-04 | 2661.4ms/step | 24.0 samples/s
578
+ Step 4570/10005 | Loss: 3.3463 | LR: 2.88e-04 | 2660.8ms/step | 24.1 samples/s
579
+ Step 4580/10005 | Loss: 3.3581 | LR: 2.87e-04 | 2660.9ms/step | 24.1 samples/s
580
+ Step 4590/10005 | Loss: 3.3381 | LR: 2.87e-04 | 2660.5ms/step | 24.1 samples/s
581
+ Step 4600/10005 | Loss: 3.3214 | LR: 2.86e-04 | 2657.0ms/step | 24.1 samples/s
582
+ Step 4610/10005 | Loss: 3.3653 | LR: 2.85e-04 | 2659.6ms/step | 24.1 samples/s
583
+ Step 4620/10005 | Loss: 3.3573 | LR: 2.84e-04 | 2660.4ms/step | 24.1 samples/s
584
+ Step 4630/10005 | Loss: 3.3505 | LR: 2.83e-04 | 2658.2ms/step | 24.1 samples/s
585
+ Step 4640/10005 | Loss: 3.3752 | LR: 2.83e-04 | 2662.3ms/step | 24.0 samples/s
586
+ Step 4650/10005 | Loss: 3.3380 | LR: 2.82e-04 | 2657.9ms/step | 24.1 samples/s
587
+ Step 4660/10005 | Loss: 3.3542 | LR: 2.81e-04 | 2660.0ms/step | 24.1 samples/s
588
+ Step 4670/10005 | Loss: 3.3598 | LR: 2.80e-04 | 2664.0ms/step | 24.0 samples/s
589
+ Step 4680/10005 | Loss: 3.3442 | LR: 2.79e-04 | 2659.3ms/step | 24.1 samples/s
590
+ Step 4690/10005 | Loss: 3.3457 | LR: 2.79e-04 | 2658.1ms/step | 24.1 samples/s
591
+ Step 4700/10005 | Loss: 3.3362 | LR: 2.78e-04 | 2660.1ms/step | 24.1 samples/s
592
+ Step 4710/10005 | Loss: 3.3645 | LR: 2.77e-04 | 2659.4ms/step | 24.1 samples/s
593
+ Step 4720/10005 | Loss: 3.3350 | LR: 2.76e-04 | 2660.7ms/step | 24.1 samples/s
594
+ Step 4730/10005 | Loss: 3.3336 | LR: 2.76e-04 | 2659.8ms/step | 24.1 samples/s
595
+ Step 4740/10005 | Loss: 3.3570 | LR: 2.75e-04 | 2661.3ms/step | 24.0 samples/s
596
+ Step 4750/10005 | Loss: 3.3335 | LR: 2.74e-04 | 2661.5ms/step | 24.0 samples/s
597
+ Validation at step 4750: Loss = 3.3403, Perplexity = 28.23
598
+ Step 4760/10005 | Loss: 3.3554 | LR: 2.73e-04 | 51345.8ms/step | 1.2 samples/s
599
+ Step 4770/10005 | Loss: 3.3541 | LR: 2.72e-04 | 2663.4ms/step | 24.0 samples/s
600
+ Step 4780/10005 | Loss: 3.3246 | LR: 2.72e-04 | 2661.5ms/step | 24.0 samples/s
601
+ Step 4790/10005 | Loss: 3.3568 | LR: 2.71e-04 | 2661.6ms/step | 24.0 samples/s
602
+ Step 4800/10005 | Loss: 3.3355 | LR: 2.70e-04 | 2662.7ms/step | 24.0 samples/s
603
+ Step 4810/10005 | Loss: 3.3474 | LR: 2.69e-04 | 2662.7ms/step | 24.0 samples/s
604
+ Step 4820/10005 | Loss: 3.3253 | LR: 2.68e-04 | 2662.7ms/step | 24.0 samples/s
605
+ Step 4830/10005 | Loss: 3.3311 | LR: 2.68e-04 | 2663.2ms/step | 24.0 samples/s
606
+ Step 4840/10005 | Loss: 3.3463 | LR: 2.67e-04 | 2659.1ms/step | 24.1 samples/s
607
+ Step 4850/10005 | Loss: 3.3182 | LR: 2.66e-04 | 2662.3ms/step | 24.0 samples/s
608
+ Step 4860/10005 | Loss: 3.3219 | LR: 2.65e-04 | 2660.1ms/step | 24.1 samples/s
609
+ Step 4870/10005 | Loss: 3.3230 | LR: 2.64e-04 | 2660.7ms/step | 24.1 samples/s
610
+ Step 4880/10005 | Loss: 3.3195 | LR: 2.64e-04 | 2663.6ms/step | 24.0 samples/s
611
+ Saved checkpoint to checkpoints/run_20260430_153357/checkpoint-4882
epfl_final/wandb/run-20260430_153400-3fhhow4v/files/requirements.txt ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ menuinst==2.4.2
2
+ anaconda-anon-usage==0.7.6
3
+ annotated-types==0.6.0
4
+ archspec==0.2.5
5
+ boltons==25.0.0
6
+ certifi==2026.4.22
7
+ charset-normalizer==3.4.4
8
+ click==8.2.1
9
+ distro==1.9.0
10
+ frozendict==2.4.6
11
+ h11==0.16.0
12
+ idna==3.11
13
+ jaraco.context==6.1.0
14
+ jeepney==0.7.1
15
+ jsonpointer==3.1.1
16
+ libmambapy==2.3.2
17
+ mdurl==0.1.2
18
+ more-itertools==11.0.2
19
+ msgpack==1.1.1
20
+ packaging==26.0
21
+ pkce==1.0.3
22
+ platformdirs==4.9.4
23
+ pluggy==1.6.0
24
+ pycosat==0.6.6
25
+ pycparser==3.0
26
+ Pygments==2.20.0
27
+ PyJWT==2.12.1
28
+ PySocks==1.7.1
29
+ python-dotenv==1.2.1
30
+ readchar==4.2.1
31
+ ruamel.yaml.clib==0.2.14
32
+ semver==3.0.4
33
+ shellingham==1.5.4
34
+ tomli==2.4.0
35
+ tomlkit==0.13.3
36
+ tqdm==4.67.3
37
+ truststore==0.10.1
38
+ typing_extensions==4.15.0
39
+ anyio==4.12.1
40
+ cffi==1.17.1
41
+ httpcore==1.0.9
42
+ jaraco.classes==3.4.0
43
+ jaraco.functools==4.4.0
44
+ jsonpatch==1.33
45
+ markdown-it-py==4.0.0
46
+ ruamel.yaml==0.18.16
47
+ typer-slim==0.20.0
48
+ typing-inspection==0.4.2
49
+ wheel==0.46.3
50
+ brotlicffi==1.2.0.0
51
+ cryptography==45.0.7
52
+ httpx==0.28.1
53
+ pip==26.0.1
54
+ pydantic_core==2.46.2
55
+ rich==14.2.0
56
+ zstandard==0.25.0
57
+ pydantic==2.13.2
58
+ SecretStorage==3.5.0
59
+ urllib3==2.6.3
60
+ keyring==25.7.0
61
+ pydantic-settings==2.12.0
62
+ requests==2.33.1
63
+ typer==0.20.0
64
+ anaconda-cli-base==0.8.2
65
+ conda_package_streaming==0.12.0
66
+ anaconda-auth==0.14.2
67
+ conda-package-handling==2.4.0
68
+ conda==26.3.2
69
+ conda-anaconda-telemetry==0.3.0
70
+ conda-anaconda-tos==0.2.2
71
+ conda-content-trust==0.3.1
72
+ conda-libmamba-solver==26.4.0
73
+ smmap==5.0.3
74
+ sentry-sdk==2.58.0
75
+ PyYAML==6.0.3
76
+ protobuf==7.34.1
77
+ gitdb==4.0.12
78
+ GitPython==3.1.49
79
+ wandb==0.26.1
80
+ nvidia-cusparselt-cu13==0.8.0
81
+ mpmath==1.3.0
82
+ cuda-toolkit==13.0.2
83
+ xxhash==3.7.0
84
+ triton==3.6.0
85
+ torchinfo==1.8.0
86
+ sympy==1.14.0
87
+ six==1.17.0
88
+ setuptools==81.0.0
89
+ safetensors==0.7.0
90
+ regex==2026.4.4
91
+ pyarrow==24.0.0
92
+ psutil==7.2.2
93
+ propcache==0.4.1
94
+ nvidia-nvtx==13.0.85
95
+ nvidia-nvshmem-cu13==3.4.5
96
+ nvidia-nvjitlink==13.0.88
97
+ nvidia-nccl-cu13==2.28.9
98
+ nvidia-curand==10.4.0.35
99
+ nvidia-cufile==1.15.1.6
100
+ nvidia-cuda-runtime==13.0.96
101
+ nvidia-cuda-nvrtc==13.0.88
102
+ nvidia-cuda-cupti==13.0.85
103
+ nvidia-cublas==13.1.0.3
104
+ numpy==2.4.4
105
+ networkx==3.6.1
106
+ multidict==6.7.1
107
+ MarkupSafe==3.0.3
108
+ hf-xet==1.4.3
109
+ fsspec==2026.2.0
110
+ frozenlist==1.8.0
111
+ filelock==3.29.0
112
+ dill==0.4.1
113
+ cuda-pathfinder==1.5.4
114
+ attrs==26.1.0
115
+ aiohappyeyeballs==2.6.1
116
+ yarl==1.23.0
117
+ python-dateutil==2.9.0.post0
118
+ nvidia-cusparse==12.6.3.3
119
+ nvidia-cufft==12.0.0.61
120
+ nvidia-cudnn-cu13==9.19.0.56
121
+ multiprocess==0.70.19
122
+ Jinja2==3.1.6
123
+ cuda-bindings==13.2.0
124
+ aiosignal==1.4.0
125
+ pandas==3.0.2
126
+ nvidia-cusolver==12.0.4.66
127
+ aiohttp==3.13.5
128
+ huggingface_hub==1.12.2
129
+ torch==2.11.0
130
+ tokenizers==0.22.2
131
+ datasets==4.8.5
132
+ transformers==5.7.0
133
+ accelerate==1.13.0
epfl_final/wandb/run-20260430_153400-3fhhow4v/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-04-30T15:34:00.733112Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-10-0-6",
11
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6713334460416"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "hrr9zq6haocwqq6tjzwaw1cgpg3n7jv2"
36
+ }
epfl_final/wandb/run-20260430_153400-3fhhow4v/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"_wandb":{"runtime":22826},"speed/samples_per_sec":24.027355057208343,"train/epoch":0,"train/learning_rate":0.0002636712368582849,"eval/perplexity":28.228371143458876,"_runtime":22826.436738721,"train/loss":3.3195291578769686,"_step":4880,"eval/loss":3.3403275411128996,"train/global_step":4880,"speed/ms_per_step":2663.6306762695312,"_timestamp":1.7775860579247587e+09,"speed/steps_per_sec":0.37542742276888036}
epfl_final/wandb/run-20260430_153400-3fhhow4v/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260430_153400-3fhhow4v/logs/debug.log ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-04-30 15:34:00,758 INFO MainThread:21222 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-04-30 15:34:00,758 INFO MainThread:21222 [wandb_setup.py:_flush():81] Configure stats pid to 21222
3
+ 2026-04-30 15:34:00,758 INFO MainThread:21222 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-04-30 15:34:00,758 INFO MainThread:21222 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_153400-3fhhow4v/logs/debug.log
5
+ 2026-04-30 15:34:00,759 INFO MainThread:21222 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_153400-3fhhow4v/logs/debug-internal.log
6
+ 2026-04-30 15:34:00,759 INFO MainThread:21222 [wandb_init.py:init():850] calling init triggers
7
+ 2026-04-30 15:34:00,759 INFO MainThread:21222 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 3840, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_153357', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10005, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 9, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-04-30 15:34:00,759 INFO MainThread:21222 [wandb_init.py:init():898] starting backend
10
+ 2026-04-30 15:34:01,057 INFO MainThread:21222 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-04-30 15:34:01,415 INFO MainThread:21222 [wandb_init.py:init():918] backend started and connected
12
+ 2026-04-30 15:34:01,424 INFO MainThread:21222 [wandb_init.py:init():988] updated telemetry
13
+ 2026-04-30 15:34:01,425 INFO MainThread:21222 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-04-30 15:34:02,398 INFO MainThread:21222 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-04-30 15:34:02,683 INFO MainThread:21222 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-04-30 15:34:02,684 INFO MainThread:21222 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-04-30 15:34:02,684 INFO MainThread:21222 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-04-30 15:34:02,684 INFO MainThread:21222 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-04-30 15:34:02,714 INFO MainThread:21222 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-04-30 21:54:28,833 INFO MainThread:21222 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/3fhhow4v
21
+ 2026-04-30 21:54:28,834 INFO MainThread:21222 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
22
+ 2026-04-30 21:54:28,835 INFO MainThread:21222 [wandb_run.py:_restore():2502] restore
23
+ 2026-04-30 21:54:28,835 INFO MainThread:21222 [wandb_run.py:_restore():2508] restore done
24
+ 2026-04-30 21:54:30,300 INFO MainThread:21222 [wandb_run.py:_footer_sync_info():3889] logging synced files
epfl_final/wandb/run-20260430_153400-3fhhow4v/run-3fhhow4v.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9762f6059845b85b002d3d8bc4d43867b3cf69aebf74ccef0c2ad74a1e10d72
3
+ size 3441233
epfl_final/wandb/run-20260430_154427-zytgzzs7/files/config.yaml ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ 3utjvuetabvi926o8bezxiyicqfp65jq:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "6053369475072"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /usr/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9
25
+ host: h100-workspace-0-6
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-04-30T15:44:27.541055Z"
33
+ writerId: 3utjvuetabvi926o8bezxiyicqfp65jq
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 49
40
+ - 51
41
+ "2":
42
+ - 1
43
+ - 11
44
+ - 49
45
+ - 51
46
+ - 71
47
+ "3":
48
+ - 2
49
+ - 15
50
+ - 16
51
+ - 61
52
+ "4": 3.10.12
53
+ "5": 0.26.1
54
+ "12": 0.26.1
55
+ "13": linux-x86_64
56
+ attention_type:
57
+ value: dense
58
+ checkpoint_activations:
59
+ value: false
60
+ eval_batches:
61
+ value: 500
62
+ eval_steps:
63
+ value: 250
64
+ gradient_accumulation_steps:
65
+ value: 8
66
+ init_from_scratch:
67
+ value: true
68
+ learning_rate:
69
+ value: 0.0005
70
+ logging_steps:
71
+ value: 10
72
+ lr_scheduler:
73
+ value: cosine
74
+ max_grad_norm:
75
+ value: 1
76
+ max_seq_length:
77
+ value: 3840
78
+ max_steps:
79
+ value: 10005
80
+ mixed_precision:
81
+ value: bf16
82
+ model_dtype:
83
+ value: bfloat16
84
+ model_path:
85
+ value: /mloscratch/homes/navasard/qwen3_06b_base
86
+ num_epochs:
87
+ value: 1
88
+ num_workers:
89
+ value: 8
90
+ output_dir:
91
+ value: ./checkpoints/run_20260430_154424
92
+ per_device_batch_size:
93
+ value: 8
94
+ resume_from_checkpoint:
95
+ value: null
96
+ save_steps:
97
+ value: 2500
98
+ seed:
99
+ value: 42
100
+ sparse_block_size:
101
+ value: 128
102
+ sparse_layer_indices:
103
+ value: null
104
+ sparse_mask_pool_size:
105
+ value: 1000
106
+ sparse_num_random_blocks:
107
+ value: 9
108
+ sparse_val_dense_compare:
109
+ value: false
110
+ strategy:
111
+ value: ddp
112
+ train_path:
113
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
114
+ val_path:
115
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
116
+ verbose:
117
+ value: true
118
+ wandb_enabled:
119
+ value: true
120
+ wandb_name:
121
+ value: null
122
+ wandb_project:
123
+ value: stochastic_sparsity
124
+ wandb_tags:
125
+ value:
126
+ - qwen3
127
+ - 0.6b
128
+ warmup_steps:
129
+ value: 100
130
+ weight_decay:
131
+ value: 0.01
epfl_final/wandb/run-20260430_154427-zytgzzs7/files/output.log ADDED
@@ -0,0 +1,598 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
2
+ Initializing model from scratch (random weights)...
3
+ Checkpoint activations: False
4
+
5
+
6
+ ================================================================================
7
+ MODEL ARCHITECTURE
8
+ ================================================================================
9
+ Model config:
10
+ vocab_size: 151936
11
+ hidden_size: 1024
12
+ num_hidden_layers: 28
13
+ num_attention_heads: 16
14
+ num_key_value_heads: 8
15
+ head_dim: 128
16
+ intermediate_size: 3072
17
+ max_position_embeddings: 32768
18
+ rope_theta: 1000000
19
+ attention_type: dense
20
+
21
+ Parameter count:
22
+ Total parameters: 596,049,920
23
+ Trainable parameters: 596,049,920
24
+ Non-trainable parameters: 0
25
+ Model size (BF16): 1.19 GB
26
+
27
+ --------------------------------------------------------------------------------
28
+ LAYER-BY-LAYER SUMMARY
29
+ --------------------------------------------------------------------------------
30
+ ===========================================================================================================================================================
31
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
32
+ ===========================================================================================================================================================
33
+ Qwen3ForCausalLM [8, 3840] -- -- True
34
+ ├─Qwen3Model: 1-1 -- [8, 3840, 1024] -- True
35
+ │ └─Embedding: 2-1 [8, 3840] [8, 3840, 1024] 155,582,464 True
36
+ │ └─ModuleList: 2-2 -- -- -- True
37
+ │ │ └─Qwen3DecoderLayer: 3-1 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
38
+ │ │ └─Qwen3DecoderLayer: 3-2 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
39
+ │ │ └─Qwen3DecoderLayer: 3-3 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
40
+ │ │ └─Qwen3DecoderLayer: 3-4 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
41
+ │ │ └─Qwen3DecoderLayer: 3-5 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
42
+ │ │ └─Qwen3DecoderLayer: 3-6 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
43
+ │ │ └─Qwen3DecoderLayer: 3-7 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
44
+ │ │ └─Qwen3DecoderLayer: 3-8 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
45
+ │ │ └─Qwen3DecoderLayer: 3-9 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
46
+ │ │ └─Qwen3DecoderLayer: 3-10 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
47
+ │ │ └─Qwen3DecoderLayer: 3-11 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
48
+ │ │ └─Qwen3DecoderLayer: 3-12 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
49
+ │ │ └─Qwen3DecoderLayer: 3-13 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
50
+ │ │ └─Qwen3DecoderLayer: 3-14 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-15 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-16 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-17 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-18 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-19 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-20 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-21 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-22 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-23 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-24 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-25 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-26 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-27 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-28 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
65
+ │ └─Qwen3RMSNorm: 2-3 [8, 3840, 1024] [8, 3840, 1024] 1,024 True
66
+ ├─Linear: 1-2 [8, 3840, 1024] [8, 3840, 151936] 155,582,464 True
67
+ ===========================================================================================================================================================
68
+ Total params: 751,632,384
69
+ Trainable params: 751,632,384
70
+ Non-trainable params: 0
71
+ Total mult-adds (G): 6.01
72
+ ===========================================================================================================================================================
73
+ Input size (MB): 0.25
74
+ Forward/backward pass size (MB): 78816.22
75
+ Params size (MB): 1503.26
76
+ Estimated Total Size (MB): 80319.73
77
+ ===========================================================================================================================================================
78
+ ================================================================================
79
+
80
+ Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
81
+ Using pre-tokenized dataset (no runtime tokenization)
82
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
83
+ Loaded 781148 pre-tokenized examples (stored_len=2048, pack_factor=2, packed_examples=390574)
84
+ Dataset split: 312459 train, 78115 validation
85
+ Training examples: 312459
86
+ Validation examples: 78115
87
+ Steps per epoch: 4882
88
+ Total training steps: 10005
89
+ Starting training...
90
+ Step 10/10005 | Loss: 11.8856 | LR: 5.00e-05 | 11176.1ms/step | 5.7 samples/s
91
+ Step 20/10005 | Loss: 10.6530 | LR: 1.00e-04 | 4571.0ms/step | 14.0 samples/s
92
+ Step 30/10005 | Loss: 9.6670 | LR: 1.50e-04 | 4565.9ms/step | 14.0 samples/s
93
+ Step 40/10005 | Loss: 8.5359 | LR: 2.00e-04 | 4565.0ms/step | 14.0 samples/s
94
+ Step 50/10005 | Loss: 8.0140 | LR: 2.50e-04 | 4570.8ms/step | 14.0 samples/s
95
+ Step 60/10005 | Loss: 7.8930 | LR: 3.00e-04 | 4579.8ms/step | 14.0 samples/s
96
+ Step 70/10005 | Loss: 7.7103 | LR: 3.50e-04 | 4586.1ms/step | 14.0 samples/s
97
+ Step 80/10005 | Loss: 7.5286 | LR: 4.00e-04 | 4592.6ms/step | 13.9 samples/s
98
+ Step 90/10005 | Loss: 7.3105 | LR: 4.50e-04 | 4596.2ms/step | 13.9 samples/s
99
+ Step 100/10005 | Loss: 7.1044 | LR: 5.00e-04 | 4600.1ms/step | 13.9 samples/s
100
+ Step 110/10005 | Loss: 6.9563 | LR: 5.00e-04 | 4600.7ms/step | 13.9 samples/s
101
+ Step 120/10005 | Loss: 6.8802 | LR: 5.00e-04 | 4601.0ms/step | 13.9 samples/s
102
+ Step 130/10005 | Loss: 6.7570 | LR: 5.00e-04 | 4600.0ms/step | 13.9 samples/s
103
+ Step 140/10005 | Loss: 6.6830 | LR: 5.00e-04 | 4600.6ms/step | 13.9 samples/s
104
+ Step 150/10005 | Loss: 6.6068 | LR: 5.00e-04 | 4602.0ms/step | 13.9 samples/s
105
+ Step 160/10005 | Loss: 6.5059 | LR: 5.00e-04 | 4600.2ms/step | 13.9 samples/s
106
+ Step 170/10005 | Loss: 6.4459 | LR: 5.00e-04 | 4733.3ms/step | 13.5 samples/s
107
+ Step 180/10005 | Loss: 6.3636 | LR: 5.00e-04 | 4610.7ms/step | 13.9 samples/s
108
+ Step 190/10005 | Loss: 6.2916 | LR: 5.00e-04 | 4615.3ms/step | 13.9 samples/s
109
+ Step 200/10005 | Loss: 6.2187 | LR: 5.00e-04 | 4614.7ms/step | 13.9 samples/s
110
+ Step 210/10005 | Loss: 6.1886 | LR: 5.00e-04 | 4616.6ms/step | 13.9 samples/s
111
+ Step 220/10005 | Loss: 6.1244 | LR: 5.00e-04 | 4617.9ms/step | 13.9 samples/s
112
+ Step 230/10005 | Loss: 6.0317 | LR: 5.00e-04 | 4618.6ms/step | 13.9 samples/s
113
+ Step 240/10005 | Loss: 5.9868 | LR: 5.00e-04 | 4608.4ms/step | 13.9 samples/s
114
+ Step 250/10005 | Loss: 5.9490 | LR: 5.00e-04 | 4616.0ms/step | 13.9 samples/s
115
+ Validation at step 250: Loss = 5.9110, Perplexity = 369.09
116
+ Step 260/10005 | Loss: 5.8882 | LR: 5.00e-04 | 78738.0ms/step | 0.8 samples/s
117
+ Step 270/10005 | Loss: 5.8132 | LR: 5.00e-04 | 4624.6ms/step | 13.8 samples/s
118
+ Step 280/10005 | Loss: 5.7704 | LR: 5.00e-04 | 4623.4ms/step | 13.8 samples/s
119
+ Step 290/10005 | Loss: 5.7240 | LR: 5.00e-04 | 4625.9ms/step | 13.8 samples/s
120
+ Step 300/10005 | Loss: 5.6844 | LR: 4.99e-04 | 4625.5ms/step | 13.8 samples/s
121
+ Step 310/10005 | Loss: 5.6032 | LR: 4.99e-04 | 4624.7ms/step | 13.8 samples/s
122
+ Step 320/10005 | Loss: 5.5698 | LR: 4.99e-04 | 4623.5ms/step | 13.8 samples/s
123
+ Step 330/10005 | Loss: 5.5386 | LR: 4.99e-04 | 4619.5ms/step | 13.9 samples/s
124
+ Step 340/10005 | Loss: 5.4798 | LR: 4.99e-04 | 4622.8ms/step | 13.8 samples/s
125
+ Step 350/10005 | Loss: 5.4517 | LR: 4.99e-04 | 4622.4ms/step | 13.8 samples/s
126
+ Step 360/10005 | Loss: 5.4517 | LR: 4.99e-04 | 4621.4ms/step | 13.8 samples/s
127
+ Step 370/10005 | Loss: 5.3707 | LR: 4.99e-04 | 4624.4ms/step | 13.8 samples/s
128
+ Step 380/10005 | Loss: 5.3079 | LR: 4.99e-04 | 4619.6ms/step | 13.9 samples/s
129
+ Step 390/10005 | Loss: 5.2689 | LR: 4.99e-04 | 4622.5ms/step | 13.8 samples/s
130
+ Step 400/10005 | Loss: 5.2015 | LR: 4.99e-04 | 4618.5ms/step | 13.9 samples/s
131
+ Step 410/10005 | Loss: 5.1538 | LR: 4.99e-04 | 4621.0ms/step | 13.8 samples/s
132
+ Step 420/10005 | Loss: 5.1201 | LR: 4.99e-04 | 4618.5ms/step | 13.9 samples/s
133
+ Step 430/10005 | Loss: 5.0644 | LR: 4.99e-04 | 4621.8ms/step | 13.8 samples/s
134
+ Step 440/10005 | Loss: 5.0290 | LR: 4.99e-04 | 4617.4ms/step | 13.9 samples/s
135
+ Step 450/10005 | Loss: 4.9837 | LR: 4.98e-04 | 4624.7ms/step | 13.8 samples/s
136
+ Step 460/10005 | Loss: 4.9486 | LR: 4.98e-04 | 4618.6ms/step | 13.9 samples/s
137
+ Step 470/10005 | Loss: 4.8924 | LR: 4.98e-04 | 4618.6ms/step | 13.9 samples/s
138
+ Step 480/10005 | Loss: 4.8702 | LR: 4.98e-04 | 4620.7ms/step | 13.9 samples/s
139
+ Step 490/10005 | Loss: 4.8186 | LR: 4.98e-04 | 4620.4ms/step | 13.9 samples/s
140
+ Step 500/10005 | Loss: 4.7803 | LR: 4.98e-04 | 4621.6ms/step | 13.8 samples/s
141
+ Validation at step 500: Loss = 4.7794, Perplexity = 119.03
142
+ Step 510/10005 | Loss: 4.7617 | LR: 4.98e-04 | 76618.8ms/step | 0.8 samples/s
143
+ Step 520/10005 | Loss: 4.7358 | LR: 4.98e-04 | 4615.4ms/step | 13.9 samples/s
144
+ Step 530/10005 | Loss: 4.7298 | LR: 4.98e-04 | 4613.4ms/step | 13.9 samples/s
145
+ Step 540/10005 | Loss: 4.6804 | LR: 4.98e-04 | 4614.6ms/step | 13.9 samples/s
146
+ Step 550/10005 | Loss: 4.6454 | LR: 4.97e-04 | 4614.1ms/step | 13.9 samples/s
147
+ Step 560/10005 | Loss: 4.6394 | LR: 4.97e-04 | 4614.6ms/step | 13.9 samples/s
148
+ Step 570/10005 | Loss: 4.6078 | LR: 4.97e-04 | 4611.5ms/step | 13.9 samples/s
149
+ Step 580/10005 | Loss: 4.5984 | LR: 4.97e-04 | 4612.9ms/step | 13.9 samples/s
150
+ Step 590/10005 | Loss: 4.5492 | LR: 4.97e-04 | 4617.7ms/step | 13.9 samples/s
151
+ Step 600/10005 | Loss: 4.5350 | LR: 4.97e-04 | 4612.4ms/step | 13.9 samples/s
152
+ Step 610/10005 | Loss: 4.5101 | LR: 4.97e-04 | 4614.2ms/step | 13.9 samples/s
153
+ Step 620/10005 | Loss: 4.5337 | LR: 4.97e-04 | 4614.4ms/step | 13.9 samples/s
154
+ Step 630/10005 | Loss: 4.5013 | LR: 4.96e-04 | 4617.5ms/step | 13.9 samples/s
155
+ Step 640/10005 | Loss: 4.4469 | LR: 4.96e-04 | 4614.4ms/step | 13.9 samples/s
156
+ Step 650/10005 | Loss: 4.4439 | LR: 4.96e-04 | 4611.6ms/step | 13.9 samples/s
157
+ Step 660/10005 | Loss: 4.4351 | LR: 4.96e-04 | 4617.2ms/step | 13.9 samples/s
158
+ Step 670/10005 | Loss: 4.4050 | LR: 4.96e-04 | 4669.6ms/step | 13.7 samples/s
159
+ Step 680/10005 | Loss: 4.3922 | LR: 4.96e-04 | 4604.0ms/step | 13.9 samples/s
160
+ Step 690/10005 | Loss: 4.3803 | LR: 4.96e-04 | 4607.1ms/step | 13.9 samples/s
161
+ Step 700/10005 | Loss: 4.3694 | LR: 4.95e-04 | 4605.8ms/step | 13.9 samples/s
162
+ Step 710/10005 | Loss: 4.3559 | LR: 4.95e-04 | 4603.5ms/step | 13.9 samples/s
163
+ Step 720/10005 | Loss: 4.3391 | LR: 4.95e-04 | 4607.8ms/step | 13.9 samples/s
164
+ Step 730/10005 | Loss: 4.3156 | LR: 4.95e-04 | 4607.9ms/step | 13.9 samples/s
165
+ Step 740/10005 | Loss: 4.3301 | LR: 4.95e-04 | 4608.6ms/step | 13.9 samples/s
166
+ Step 750/10005 | Loss: 4.2846 | LR: 4.95e-04 | 4605.0ms/step | 13.9 samples/s
167
+ Validation at step 750: Loss = 4.2874, Perplexity = 72.78
168
+ Step 760/10005 | Loss: 4.2533 | LR: 4.95e-04 | 76609.6ms/step | 0.8 samples/s
169
+ Step 770/10005 | Loss: 4.2656 | LR: 4.94e-04 | 4605.8ms/step | 13.9 samples/s
170
+ Step 780/10005 | Loss: 4.2494 | LR: 4.94e-04 | 4607.5ms/step | 13.9 samples/s
171
+ Step 790/10005 | Loss: 4.2337 | LR: 4.94e-04 | 4606.7ms/step | 13.9 samples/s
172
+ Step 800/10005 | Loss: 4.2200 | LR: 4.94e-04 | 4601.0ms/step | 13.9 samples/s
173
+ Step 810/10005 | Loss: 4.2072 | LR: 4.94e-04 | 4605.1ms/step | 13.9 samples/s
174
+ Step 820/10005 | Loss: 4.2127 | LR: 4.94e-04 | 4608.6ms/step | 13.9 samples/s
175
+ Step 830/10005 | Loss: 4.1919 | LR: 4.93e-04 | 4609.2ms/step | 13.9 samples/s
176
+ Step 840/10005 | Loss: 4.1769 | LR: 4.93e-04 | 4606.4ms/step | 13.9 samples/s
177
+ Step 850/10005 | Loss: 4.1734 | LR: 4.93e-04 | 4607.3ms/step | 13.9 samples/s
178
+ Step 860/10005 | Loss: 4.1637 | LR: 4.93e-04 | 4604.7ms/step | 13.9 samples/s
179
+ Step 870/10005 | Loss: 4.1614 | LR: 4.93e-04 | 4603.8ms/step | 13.9 samples/s
180
+ Step 880/10005 | Loss: 4.1437 | LR: 4.92e-04 | 4618.7ms/step | 13.9 samples/s
181
+ Step 890/10005 | Loss: 4.1498 | LR: 4.92e-04 | 4610.8ms/step | 13.9 samples/s
182
+ Step 900/10005 | Loss: 4.1046 | LR: 4.92e-04 | 4606.5ms/step | 13.9 samples/s
183
+ Step 910/10005 | Loss: 4.0953 | LR: 4.92e-04 | 4607.5ms/step | 13.9 samples/s
184
+ Step 920/10005 | Loss: 4.1211 | LR: 4.92e-04 | 4607.1ms/step | 13.9 samples/s
185
+ Step 930/10005 | Loss: 4.1054 | LR: 4.91e-04 | 4606.2ms/step | 13.9 samples/s
186
+ Step 940/10005 | Loss: 4.0779 | LR: 4.91e-04 | 4605.8ms/step | 13.9 samples/s
187
+ Step 950/10005 | Loss: 4.0819 | LR: 4.91e-04 | 4603.5ms/step | 13.9 samples/s
188
+ Step 960/10005 | Loss: 4.0550 | LR: 4.91e-04 | 4603.4ms/step | 13.9 samples/s
189
+ Step 970/10005 | Loss: 4.0561 | LR: 4.91e-04 | 4605.0ms/step | 13.9 samples/s
190
+ Step 980/10005 | Loss: 4.0692 | LR: 4.90e-04 | 4604.3ms/step | 13.9 samples/s
191
+ Step 990/10005 | Loss: 4.0352 | LR: 4.90e-04 | 4605.2ms/step | 13.9 samples/s
192
+ Step 1000/10005 | Loss: 4.0510 | LR: 4.90e-04 | 4610.9ms/step | 13.9 samples/s
193
+ Validation at step 1000: Loss = 4.0346, Perplexity = 56.52
194
+ Step 1010/10005 | Loss: 4.0168 | LR: 4.90e-04 | 76604.6ms/step | 0.8 samples/s
195
+ Step 1020/10005 | Loss: 4.0317 | LR: 4.89e-04 | 4609.2ms/step | 13.9 samples/s
196
+ Step 1030/10005 | Loss: 4.0027 | LR: 4.89e-04 | 4608.9ms/step | 13.9 samples/s
197
+ Step 1040/10005 | Loss: 4.0096 | LR: 4.89e-04 | 4610.1ms/step | 13.9 samples/s
198
+ Step 1050/10005 | Loss: 4.0063 | LR: 4.89e-04 | 4612.3ms/step | 13.9 samples/s
199
+ Step 1060/10005 | Loss: 3.9727 | LR: 4.89e-04 | 4628.2ms/step | 13.8 samples/s
200
+ Step 1070/10005 | Loss: 3.9721 | LR: 4.88e-04 | 4612.2ms/step | 13.9 samples/s
201
+ Step 1080/10005 | Loss: 3.9873 | LR: 4.88e-04 | 4607.3ms/step | 13.9 samples/s
202
+ Step 1090/10005 | Loss: 3.9905 | LR: 4.88e-04 | 4610.8ms/step | 13.9 samples/s
203
+ Step 1100/10005 | Loss: 3.9414 | LR: 4.88e-04 | 4609.8ms/step | 13.9 samples/s
204
+ Step 1110/10005 | Loss: 3.9749 | LR: 4.87e-04 | 4608.9ms/step | 13.9 samples/s
205
+ Step 1120/10005 | Loss: 3.9548 | LR: 4.87e-04 | 4610.6ms/step | 13.9 samples/s
206
+ Step 1130/10005 | Loss: 3.9224 | LR: 4.87e-04 | 4610.0ms/step | 13.9 samples/s
207
+ Step 1140/10005 | Loss: 3.9302 | LR: 4.87e-04 | 4730.7ms/step | 13.5 samples/s
208
+ Step 1150/10005 | Loss: 3.9275 | LR: 4.86e-04 | 4608.6ms/step | 13.9 samples/s
209
+ Step 1160/10005 | Loss: 3.9261 | LR: 4.86e-04 | 4607.4ms/step | 13.9 samples/s
210
+ Step 1170/10005 | Loss: 3.9337 | LR: 4.86e-04 | 4609.1ms/step | 13.9 samples/s
211
+ Step 1180/10005 | Loss: 3.8978 | LR: 4.85e-04 | 4608.1ms/step | 13.9 samples/s
212
+ Step 1190/10005 | Loss: 3.9213 | LR: 4.85e-04 | 4605.3ms/step | 13.9 samples/s
213
+ Step 1200/10005 | Loss: 3.8809 | LR: 4.85e-04 | 4606.4ms/step | 13.9 samples/s
214
+ Step 1210/10005 | Loss: 3.9009 | LR: 4.85e-04 | 4605.6ms/step | 13.9 samples/s
215
+ Step 1220/10005 | Loss: 3.8915 | LR: 4.84e-04 | 4604.9ms/step | 13.9 samples/s
216
+ Step 1230/10005 | Loss: 3.9012 | LR: 4.84e-04 | 4609.7ms/step | 13.9 samples/s
217
+ Step 1240/10005 | Loss: 3.8738 | LR: 4.84e-04 | 4609.4ms/step | 13.9 samples/s
218
+ Step 1250/10005 | Loss: 3.8857 | LR: 4.84e-04 | 4605.2ms/step | 13.9 samples/s
219
+ Validation at step 1250: Loss = 3.8793, Perplexity = 48.39
220
+ Step 1260/10005 | Loss: 3.8989 | LR: 4.83e-04 | 76611.6ms/step | 0.8 samples/s
221
+ Step 1270/10005 | Loss: 3.8476 | LR: 4.83e-04 | 4608.3ms/step | 13.9 samples/s
222
+ Step 1280/10005 | Loss: 3.8469 | LR: 4.83e-04 | 4604.3ms/step | 13.9 samples/s
223
+ Step 1290/10005 | Loss: 3.8549 | LR: 4.82e-04 | 4604.1ms/step | 13.9 samples/s
224
+ Step 1300/10005 | Loss: 3.8610 | LR: 4.82e-04 | 4611.5ms/step | 13.9 samples/s
225
+ Step 1310/10005 | Loss: 3.8559 | LR: 4.82e-04 | 4610.3ms/step | 13.9 samples/s
226
+ Step 1320/10005 | Loss: 3.8380 | LR: 4.82e-04 | 4610.0ms/step | 13.9 samples/s
227
+ Step 1330/10005 | Loss: 3.8675 | LR: 4.81e-04 | 4611.0ms/step | 13.9 samples/s
228
+ Step 1340/10005 | Loss: 3.8286 | LR: 4.81e-04 | 4608.0ms/step | 13.9 samples/s
229
+ Step 1350/10005 | Loss: 3.8237 | LR: 4.81e-04 | 4608.7ms/step | 13.9 samples/s
230
+ Step 1360/10005 | Loss: 3.8418 | LR: 4.80e-04 | 4609.8ms/step | 13.9 samples/s
231
+ Step 1370/10005 | Loss: 3.8426 | LR: 4.80e-04 | 4608.4ms/step | 13.9 samples/s
232
+ Step 1380/10005 | Loss: 3.8378 | LR: 4.80e-04 | 4611.5ms/step | 13.9 samples/s
233
+ Step 1390/10005 | Loss: 3.7908 | LR: 4.79e-04 | 4611.3ms/step | 13.9 samples/s
234
+ Step 1400/10005 | Loss: 3.8100 | LR: 4.79e-04 | 4606.7ms/step | 13.9 samples/s
235
+ Step 1410/10005 | Loss: 3.7829 | LR: 4.79e-04 | 4609.5ms/step | 13.9 samples/s
236
+ Step 1420/10005 | Loss: 3.7937 | LR: 4.78e-04 | 4613.1ms/step | 13.9 samples/s
237
+ Step 1430/10005 | Loss: 3.7967 | LR: 4.78e-04 | 4609.7ms/step | 13.9 samples/s
238
+ Step 1440/10005 | Loss: 3.7943 | LR: 4.78e-04 | 4607.5ms/step | 13.9 samples/s
239
+ Step 1450/10005 | Loss: 3.7869 | LR: 4.77e-04 | 4609.1ms/step | 13.9 samples/s
240
+ Step 1460/10005 | Loss: 3.7924 | LR: 4.77e-04 | 4605.0ms/step | 13.9 samples/s
241
+ Step 1470/10005 | Loss: 3.8047 | LR: 4.77e-04 | 4608.8ms/step | 13.9 samples/s
242
+ Step 1480/10005 | Loss: 3.7873 | LR: 4.76e-04 | 4609.7ms/step | 13.9 samples/s
243
+ Step 1490/10005 | Loss: 3.7597 | LR: 4.76e-04 | 4609.1ms/step | 13.9 samples/s
244
+ Step 1500/10005 | Loss: 3.7826 | LR: 4.76e-04 | 4607.9ms/step | 13.9 samples/s
245
+ Validation at step 1500: Loss = 3.7716, Perplexity = 43.45
246
+ Step 1510/10005 | Loss: 3.7830 | LR: 4.75e-04 | 76658.7ms/step | 0.8 samples/s
247
+ Step 1520/10005 | Loss: 3.7468 | LR: 4.75e-04 | 4612.1ms/step | 13.9 samples/s
248
+ Step 1530/10005 | Loss: 3.7624 | LR: 4.75e-04 | 4606.2ms/step | 13.9 samples/s
249
+ Step 1540/10005 | Loss: 3.7642 | LR: 4.74e-04 | 4607.1ms/step | 13.9 samples/s
250
+ Step 1550/10005 | Loss: 3.7599 | LR: 4.74e-04 | 4610.6ms/step | 13.9 samples/s
251
+ Step 1560/10005 | Loss: 3.7618 | LR: 4.74e-04 | 4608.4ms/step | 13.9 samples/s
252
+ Step 1570/10005 | Loss: 3.7724 | LR: 4.73e-04 | 4608.9ms/step | 13.9 samples/s
253
+ Step 1580/10005 | Loss: 3.7195 | LR: 4.73e-04 | 4611.3ms/step | 13.9 samples/s
254
+ Step 1590/10005 | Loss: 3.7380 | LR: 4.73e-04 | 4609.2ms/step | 13.9 samples/s
255
+ Step 1600/10005 | Loss: 3.7054 | LR: 4.72e-04 | 4608.6ms/step | 13.9 samples/s
256
+ Step 1610/10005 | Loss: 3.7355 | LR: 4.72e-04 | 4670.8ms/step | 13.7 samples/s
257
+ Step 1620/10005 | Loss: 3.7280 | LR: 4.72e-04 | 4607.0ms/step | 13.9 samples/s
258
+ Step 1630/10005 | Loss: 3.7431 | LR: 4.71e-04 | 4607.7ms/step | 13.9 samples/s
259
+ Step 1640/10005 | Loss: 3.7143 | LR: 4.71e-04 | 4606.4ms/step | 13.9 samples/s
260
+ Step 1650/10005 | Loss: 3.7253 | LR: 4.70e-04 | 4607.3ms/step | 13.9 samples/s
261
+ Step 1660/10005 | Loss: 3.7163 | LR: 4.70e-04 | 4611.4ms/step | 13.9 samples/s
262
+ Step 1670/10005 | Loss: 3.7066 | LR: 4.70e-04 | 4605.3ms/step | 13.9 samples/s
263
+ Step 1680/10005 | Loss: 3.7164 | LR: 4.69e-04 | 4609.3ms/step | 13.9 samples/s
264
+ Step 1690/10005 | Loss: 3.7150 | LR: 4.69e-04 | 4607.6ms/step | 13.9 samples/s
265
+ Step 1700/10005 | Loss: 3.6848 | LR: 4.68e-04 | 4605.5ms/step | 13.9 samples/s
266
+ Step 1710/10005 | Loss: 3.6709 | LR: 4.68e-04 | 4609.2ms/step | 13.9 samples/s
267
+ Step 1720/10005 | Loss: 3.7011 | LR: 4.68e-04 | 4602.3ms/step | 13.9 samples/s
268
+ Step 1730/10005 | Loss: 3.6891 | LR: 4.67e-04 | 4607.8ms/step | 13.9 samples/s
269
+ Step 1740/10005 | Loss: 3.6742 | LR: 4.67e-04 | 4607.3ms/step | 13.9 samples/s
270
+ Step 1750/10005 | Loss: 3.6557 | LR: 4.67e-04 | 4604.7ms/step | 13.9 samples/s
271
+ Validation at step 1750: Loss = 3.6898, Perplexity = 40.04
272
+ Step 1760/10005 | Loss: 3.6691 | LR: 4.66e-04 | 76609.3ms/step | 0.8 samples/s
273
+ Step 1770/10005 | Loss: 3.6801 | LR: 4.66e-04 | 4608.1ms/step | 13.9 samples/s
274
+ Step 1780/10005 | Loss: 3.6857 | LR: 4.65e-04 | 4605.1ms/step | 13.9 samples/s
275
+ Step 1790/10005 | Loss: 3.6838 | LR: 4.65e-04 | 4606.2ms/step | 13.9 samples/s
276
+ Step 1800/10005 | Loss: 3.6622 | LR: 4.65e-04 | 4606.9ms/step | 13.9 samples/s
277
+ Step 1810/10005 | Loss: 3.6596 | LR: 4.64e-04 | 4609.6ms/step | 13.9 samples/s
278
+ Step 1820/10005 | Loss: 3.6484 | LR: 4.64e-04 | 4606.6ms/step | 13.9 samples/s
279
+ Step 1830/10005 | Loss: 3.6633 | LR: 4.63e-04 | 4606.4ms/step | 13.9 samples/s
280
+ Step 1840/10005 | Loss: 3.6689 | LR: 4.63e-04 | 4607.9ms/step | 13.9 samples/s
281
+ Step 1850/10005 | Loss: 3.6562 | LR: 4.62e-04 | 4608.8ms/step | 13.9 samples/s
282
+ Step 1860/10005 | Loss: 3.6515 | LR: 4.62e-04 | 4610.4ms/step | 13.9 samples/s
283
+ Step 1870/10005 | Loss: 3.6356 | LR: 4.62e-04 | 4607.3ms/step | 13.9 samples/s
284
+ Step 1880/10005 | Loss: 3.6573 | LR: 4.61e-04 | 4607.4ms/step | 13.9 samples/s
285
+ Step 1890/10005 | Loss: 3.6441 | LR: 4.61e-04 | 4605.3ms/step | 13.9 samples/s
286
+ Step 1900/10005 | Loss: 3.6498 | LR: 4.60e-04 | 4604.8ms/step | 13.9 samples/s
287
+ Step 1910/10005 | Loss: 3.6620 | LR: 4.60e-04 | 4604.3ms/step | 13.9 samples/s
288
+ Step 1920/10005 | Loss: 3.6393 | LR: 4.59e-04 | 4603.1ms/step | 13.9 samples/s
289
+ Step 1930/10005 | Loss: 3.6358 | LR: 4.59e-04 | 4603.2ms/step | 13.9 samples/s
290
+ Step 1940/10005 | Loss: 3.6414 | LR: 4.59e-04 | 4604.5ms/step | 13.9 samples/s
291
+ Step 1950/10005 | Loss: 3.6256 | LR: 4.58e-04 | 4603.8ms/step | 13.9 samples/s
292
+ Step 1960/10005 | Loss: 3.6245 | LR: 4.58e-04 | 4606.9ms/step | 13.9 samples/s
293
+ Step 1970/10005 | Loss: 3.6117 | LR: 4.57e-04 | 4603.8ms/step | 13.9 samples/s
294
+ Step 1980/10005 | Loss: 3.6198 | LR: 4.57e-04 | 4604.1ms/step | 13.9 samples/s
295
+ Step 1990/10005 | Loss: 3.6140 | LR: 4.56e-04 | 4607.8ms/step | 13.9 samples/s
296
+ Step 2000/10005 | Loss: 3.6074 | LR: 4.56e-04 | 4604.8ms/step | 13.9 samples/s
297
+ Validation at step 2000: Loss = 3.6235, Perplexity = 37.47
298
+ Step 2010/10005 | Loss: 3.5998 | LR: 4.56e-04 | 76605.7ms/step | 0.8 samples/s
299
+ Step 2020/10005 | Loss: 3.6154 | LR: 4.55e-04 | 4604.1ms/step | 13.9 samples/s
300
+ Step 2030/10005 | Loss: 3.6085 | LR: 4.55e-04 | 4603.0ms/step | 13.9 samples/s
301
+ Step 2040/10005 | Loss: 3.6106 | LR: 4.54e-04 | 4602.7ms/step | 13.9 samples/s
302
+ Step 2050/10005 | Loss: 3.6361 | LR: 4.54e-04 | 4607.2ms/step | 13.9 samples/s
303
+ Step 2060/10005 | Loss: 3.6066 | LR: 4.53e-04 | 4605.2ms/step | 13.9 samples/s
304
+ Step 2070/10005 | Loss: 3.6150 | LR: 4.53e-04 | 4603.6ms/step | 13.9 samples/s
305
+ Step 2080/10005 | Loss: 3.5982 | LR: 4.52e-04 | 4654.1ms/step | 13.8 samples/s
306
+ Step 2090/10005 | Loss: 3.5832 | LR: 4.52e-04 | 4603.2ms/step | 13.9 samples/s
307
+ Step 2100/10005 | Loss: 3.6074 | LR: 4.51e-04 | 4606.5ms/step | 13.9 samples/s
308
+ Step 2110/10005 | Loss: 3.5947 | LR: 4.51e-04 | 4605.7ms/step | 13.9 samples/s
309
+ Step 2120/10005 | Loss: 3.6109 | LR: 4.50e-04 | 4605.2ms/step | 13.9 samples/s
310
+ Step 2130/10005 | Loss: 3.5779 | LR: 4.50e-04 | 4608.6ms/step | 13.9 samples/s
311
+ Step 2140/10005 | Loss: 3.6112 | LR: 4.49e-04 | 4606.4ms/step | 13.9 samples/s
312
+ Step 2150/10005 | Loss: 3.6216 | LR: 4.49e-04 | 4606.3ms/step | 13.9 samples/s
313
+ Step 2160/10005 | Loss: 3.5874 | LR: 4.49e-04 | 4605.0ms/step | 13.9 samples/s
314
+ Step 2170/10005 | Loss: 3.5735 | LR: 4.48e-04 | 4605.8ms/step | 13.9 samples/s
315
+ Step 2180/10005 | Loss: 3.5608 | LR: 4.48e-04 | 4603.3ms/step | 13.9 samples/s
316
+ Step 2190/10005 | Loss: 3.5685 | LR: 4.47e-04 | 4602.8ms/step | 13.9 samples/s
317
+ Step 2200/10005 | Loss: 3.5808 | LR: 4.47e-04 | 4602.3ms/step | 13.9 samples/s
318
+ Step 2210/10005 | Loss: 3.5830 | LR: 4.46e-04 | 4607.0ms/step | 13.9 samples/s
319
+ Step 2220/10005 | Loss: 3.5881 | LR: 4.46e-04 | 4607.6ms/step | 13.9 samples/s
320
+ Step 2230/10005 | Loss: 3.6003 | LR: 4.45e-04 | 4606.6ms/step | 13.9 samples/s
321
+ Step 2240/10005 | Loss: 3.5948 | LR: 4.45e-04 | 4605.3ms/step | 13.9 samples/s
322
+ Step 2250/10005 | Loss: 3.5556 | LR: 4.44e-04 | 4601.5ms/step | 13.9 samples/s
323
+ Validation at step 2250: Loss = 3.5687, Perplexity = 35.47
324
+ Step 2260/10005 | Loss: 3.5742 | LR: 4.44e-04 | 76610.5ms/step | 0.8 samples/s
325
+ Step 2270/10005 | Loss: 3.5474 | LR: 4.43e-04 | 4605.2ms/step | 13.9 samples/s
326
+ Step 2280/10005 | Loss: 3.5563 | LR: 4.43e-04 | 4604.3ms/step | 13.9 samples/s
327
+ Step 2290/10005 | Loss: 3.5770 | LR: 4.42e-04 | 4606.7ms/step | 13.9 samples/s
328
+ Step 2300/10005 | Loss: 3.5585 | LR: 4.42e-04 | 4606.3ms/step | 13.9 samples/s
329
+ Step 2310/10005 | Loss: 3.5692 | LR: 4.41e-04 | 4604.9ms/step | 13.9 samples/s
330
+ Step 2320/10005 | Loss: 3.5491 | LR: 4.41e-04 | 4604.3ms/step | 13.9 samples/s
331
+ Step 2330/10005 | Loss: 3.5374 | LR: 4.40e-04 | 4606.6ms/step | 13.9 samples/s
332
+ Step 2340/10005 | Loss: 3.5446 | LR: 4.40e-04 | 4607.5ms/step | 13.9 samples/s
333
+ Step 2350/10005 | Loss: 3.5530 | LR: 4.39e-04 | 4607.6ms/step | 13.9 samples/s
334
+ Step 2360/10005 | Loss: 3.5727 | LR: 4.38e-04 | 4605.6ms/step | 13.9 samples/s
335
+ Step 2370/10005 | Loss: 3.5432 | LR: 4.38e-04 | 4605.6ms/step | 13.9 samples/s
336
+ Step 2380/10005 | Loss: 3.5644 | LR: 4.37e-04 | 4609.7ms/step | 13.9 samples/s
337
+ Step 2390/10005 | Loss: 3.5305 | LR: 4.37e-04 | 4612.5ms/step | 13.9 samples/s
338
+ Step 2400/10005 | Loss: 3.5278 | LR: 4.36e-04 | 4609.9ms/step | 13.9 samples/s
339
+ Step 2410/10005 | Loss: 3.5266 | LR: 4.36e-04 | 4607.0ms/step | 13.9 samples/s
340
+ Step 2420/10005 | Loss: 3.5366 | LR: 4.35e-04 | 4605.7ms/step | 13.9 samples/s
341
+ Step 2430/10005 | Loss: 3.5292 | LR: 4.35e-04 | 4607.6ms/step | 13.9 samples/s
342
+ Step 2440/10005 | Loss: 3.5302 | LR: 4.34e-04 | 4606.6ms/step | 13.9 samples/s
343
+ Step 2450/10005 | Loss: 3.4945 | LR: 4.34e-04 | 4605.5ms/step | 13.9 samples/s
344
+ Step 2460/10005 | Loss: 3.5260 | LR: 4.33e-04 | 4604.1ms/step | 13.9 samples/s
345
+ Step 2470/10005 | Loss: 3.5834 | LR: 4.33e-04 | 4608.9ms/step | 13.9 samples/s
346
+ Step 2480/10005 | Loss: 3.5190 | LR: 4.32e-04 | 4605.1ms/step | 13.9 samples/s
347
+ Step 2490/10005 | Loss: 3.5268 | LR: 4.32e-04 | 4609.6ms/step | 13.9 samples/s
348
+ Step 2500/10005 | Loss: 3.5188 | LR: 4.31e-04 | 4613.7ms/step | 13.9 samples/s
349
+ Validation at step 2500: Loss = 3.5211, Perplexity = 33.82
350
+ Saved checkpoint to checkpoints/run_20260430_154424/checkpoint-2500
351
+ Step 2510/10005 | Loss: 3.5296 | LR: 4.30e-04 | 77110.6ms/step | 0.8 samples/s
352
+ Step 2520/10005 | Loss: 3.5103 | LR: 4.30e-04 | 4613.9ms/step | 13.9 samples/s
353
+ Step 2530/10005 | Loss: 3.5376 | LR: 4.29e-04 | 4618.2ms/step | 13.9 samples/s
354
+ Step 2540/10005 | Loss: 3.5026 | LR: 4.29e-04 | 4621.0ms/step | 13.8 samples/s
355
+ Step 2550/10005 | Loss: 3.5173 | LR: 4.28e-04 | 4662.1ms/step | 13.7 samples/s
356
+ Step 2560/10005 | Loss: 3.5128 | LR: 4.28e-04 | 4614.1ms/step | 13.9 samples/s
357
+ Step 2570/10005 | Loss: 3.5182 | LR: 4.27e-04 | 4617.6ms/step | 13.9 samples/s
358
+ Step 2580/10005 | Loss: 3.5315 | LR: 4.27e-04 | 4615.2ms/step | 13.9 samples/s
359
+ Step 2590/10005 | Loss: 3.5097 | LR: 4.26e-04 | 4613.0ms/step | 13.9 samples/s
360
+ Step 2600/10005 | Loss: 3.5070 | LR: 4.25e-04 | 4616.7ms/step | 13.9 samples/s
361
+ Step 2610/10005 | Loss: 3.4863 | LR: 4.25e-04 | 4616.4ms/step | 13.9 samples/s
362
+ Step 2620/10005 | Loss: 3.4823 | LR: 4.24e-04 | 4611.9ms/step | 13.9 samples/s
363
+ Step 2630/10005 | Loss: 3.5187 | LR: 4.24e-04 | 4614.5ms/step | 13.9 samples/s
364
+ Step 2640/10005 | Loss: 3.4778 | LR: 4.23e-04 | 4613.2ms/step | 13.9 samples/s
365
+ Step 2650/10005 | Loss: 3.4895 | LR: 4.23e-04 | 4619.9ms/step | 13.9 samples/s
366
+ Step 2660/10005 | Loss: 3.5027 | LR: 4.22e-04 | 4617.6ms/step | 13.9 samples/s
367
+ Step 2670/10005 | Loss: 3.4988 | LR: 4.21e-04 | 4615.8ms/step | 13.9 samples/s
368
+ Step 2680/10005 | Loss: 3.4923 | LR: 4.21e-04 | 4616.2ms/step | 13.9 samples/s
369
+ Step 2690/10005 | Loss: 3.4979 | LR: 4.20e-04 | 4614.2ms/step | 13.9 samples/s
370
+ Step 2700/10005 | Loss: 3.5115 | LR: 4.20e-04 | 4614.5ms/step | 13.9 samples/s
371
+ Step 2710/10005 | Loss: 3.5062 | LR: 4.19e-04 | 4611.7ms/step | 13.9 samples/s
372
+ Step 2720/10005 | Loss: 3.4843 | LR: 4.19e-04 | 4613.9ms/step | 13.9 samples/s
373
+ Step 2730/10005 | Loss: 3.4642 | LR: 4.18e-04 | 4615.9ms/step | 13.9 samples/s
374
+ Step 2740/10005 | Loss: 3.4704 | LR: 4.17e-04 | 4613.9ms/step | 13.9 samples/s
375
+ Step 2750/10005 | Loss: 3.4861 | LR: 4.17e-04 | 4613.8ms/step | 13.9 samples/s
376
+ Validation at step 2750: Loss = 3.4852, Perplexity = 32.63
377
+ Step 2760/10005 | Loss: 3.4971 | LR: 4.16e-04 | 76621.4ms/step | 0.8 samples/s
378
+ Step 2770/10005 | Loss: 3.4958 | LR: 4.16e-04 | 4612.2ms/step | 13.9 samples/s
379
+ Step 2780/10005 | Loss: 3.4758 | LR: 4.15e-04 | 4612.0ms/step | 13.9 samples/s
380
+ Step 2790/10005 | Loss: 3.4708 | LR: 4.14e-04 | 4611.6ms/step | 13.9 samples/s
381
+ Step 2800/10005 | Loss: 3.4410 | LR: 4.14e-04 | 4609.6ms/step | 13.9 samples/s
382
+ Step 2810/10005 | Loss: 3.4710 | LR: 4.13e-04 | 4612.8ms/step | 13.9 samples/s
383
+ Step 2820/10005 | Loss: 3.4751 | LR: 4.13e-04 | 4612.5ms/step | 13.9 samples/s
384
+ Step 2830/10005 | Loss: 3.4633 | LR: 4.12e-04 | 4612.5ms/step | 13.9 samples/s
385
+ Step 2840/10005 | Loss: 3.4788 | LR: 4.11e-04 | 4611.4ms/step | 13.9 samples/s
386
+ Step 2850/10005 | Loss: 3.4337 | LR: 4.11e-04 | 4615.6ms/step | 13.9 samples/s
387
+ Step 2860/10005 | Loss: 3.4628 | LR: 4.10e-04 | 4615.1ms/step | 13.9 samples/s
388
+ Step 2870/10005 | Loss: 3.4534 | LR: 4.10e-04 | 4614.6ms/step | 13.9 samples/s
389
+ Step 2880/10005 | Loss: 3.4607 | LR: 4.09e-04 | 4612.4ms/step | 13.9 samples/s
390
+ Step 2890/10005 | Loss: 3.4552 | LR: 4.08e-04 | 4613.5ms/step | 13.9 samples/s
391
+ Step 2900/10005 | Loss: 3.4593 | LR: 4.08e-04 | 4614.4ms/step | 13.9 samples/s
392
+ Step 2910/10005 | Loss: 3.4509 | LR: 4.07e-04 | 4615.0ms/step | 13.9 samples/s
393
+ Step 2920/10005 | Loss: 3.4745 | LR: 4.06e-04 | 4613.3ms/step | 13.9 samples/s
394
+ Step 2930/10005 | Loss: 3.4434 | LR: 4.06e-04 | 4612.1ms/step | 13.9 samples/s
395
+ Step 2940/10005 | Loss: 3.4575 | LR: 4.05e-04 | 4616.3ms/step | 13.9 samples/s
396
+ Step 2950/10005 | Loss: 3.4739 | LR: 4.05e-04 | 4616.6ms/step | 13.9 samples/s
397
+ Step 2960/10005 | Loss: 3.4318 | LR: 4.04e-04 | 4617.1ms/step | 13.9 samples/s
398
+ Step 2970/10005 | Loss: 3.4501 | LR: 4.03e-04 | 4615.7ms/step | 13.9 samples/s
399
+ Step 2980/10005 | Loss: 3.4248 | LR: 4.03e-04 | 4615.4ms/step | 13.9 samples/s
400
+ Step 2990/10005 | Loss: 3.4267 | LR: 4.02e-04 | 4614.2ms/step | 13.9 samples/s
401
+ Step 3000/10005 | Loss: 3.4419 | LR: 4.01e-04 | 4621.2ms/step | 13.8 samples/s
402
+ Validation at step 3000: Loss = 3.4501, Perplexity = 31.50
403
+ Step 3010/10005 | Loss: 3.4407 | LR: 4.01e-04 | 76645.0ms/step | 0.8 samples/s
404
+ Step 3020/10005 | Loss: 3.4375 | LR: 4.00e-04 | 4712.0ms/step | 13.6 samples/s
405
+ Step 3030/10005 | Loss: 3.4356 | LR: 4.00e-04 | 4600.8ms/step | 13.9 samples/s
406
+ Step 3040/10005 | Loss: 3.4362 | LR: 3.99e-04 | 4617.7ms/step | 13.9 samples/s
407
+ Step 3050/10005 | Loss: 3.4450 | LR: 3.98e-04 | 4615.5ms/step | 13.9 samples/s
408
+ Step 3060/10005 | Loss: 3.4454 | LR: 3.98e-04 | 4599.8ms/step | 13.9 samples/s
409
+ Step 3070/10005 | Loss: 3.4371 | LR: 3.97e-04 | 4591.8ms/step | 13.9 samples/s
410
+ Step 3080/10005 | Loss: 3.4260 | LR: 3.96e-04 | 4595.1ms/step | 13.9 samples/s
411
+ Step 3090/10005 | Loss: 3.4271 | LR: 3.96e-04 | 4594.4ms/step | 13.9 samples/s
412
+ Step 3100/10005 | Loss: 3.4322 | LR: 3.95e-04 | 4612.8ms/step | 13.9 samples/s
413
+ Step 3110/10005 | Loss: 3.4446 | LR: 3.94e-04 | 4599.5ms/step | 13.9 samples/s
414
+ Step 3120/10005 | Loss: 3.4569 | LR: 3.94e-04 | 4590.6ms/step | 13.9 samples/s
415
+ Step 3130/10005 | Loss: 3.4335 | LR: 3.93e-04 | 4591.0ms/step | 13.9 samples/s
416
+ Step 3140/10005 | Loss: 3.4169 | LR: 3.93e-04 | 4589.3ms/step | 13.9 samples/s
417
+ Step 3150/10005 | Loss: 3.4282 | LR: 3.92e-04 | 4591.0ms/step | 13.9 samples/s
418
+ Step 3160/10005 | Loss: 3.4069 | LR: 3.91e-04 | 4592.1ms/step | 13.9 samples/s
419
+ Step 3170/10005 | Loss: 3.4154 | LR: 3.91e-04 | 4598.2ms/step | 13.9 samples/s
420
+ Step 3180/10005 | Loss: 3.4411 | LR: 3.90e-04 | 4590.1ms/step | 13.9 samples/s
421
+ Step 3190/10005 | Loss: 3.4306 | LR: 3.89e-04 | 4598.4ms/step | 13.9 samples/s
422
+ Step 3200/10005 | Loss: 3.4218 | LR: 3.89e-04 | 4609.7ms/step | 13.9 samples/s
423
+ Step 3210/10005 | Loss: 3.4242 | LR: 3.88e-04 | 4589.0ms/step | 13.9 samples/s
424
+ Step 3220/10005 | Loss: 3.4183 | LR: 3.87e-04 | 4601.1ms/step | 13.9 samples/s
425
+ Step 3230/10005 | Loss: 3.4117 | LR: 3.87e-04 | 4626.1ms/step | 13.8 samples/s
426
+ Step 3240/10005 | Loss: 3.4281 | LR: 3.86e-04 | 4600.8ms/step | 13.9 samples/s
427
+ Step 3250/10005 | Loss: 3.4179 | LR: 3.85e-04 | 4591.9ms/step | 13.9 samples/s
428
+ Validation at step 3250: Loss = 3.4194, Perplexity = 30.55
429
+ Step 3260/10005 | Loss: 3.4261 | LR: 3.85e-04 | 76642.9ms/step | 0.8 samples/s
430
+ Step 3270/10005 | Loss: 3.4157 | LR: 3.84e-04 | 4634.7ms/step | 13.8 samples/s
431
+ Step 3280/10005 | Loss: 3.4156 | LR: 3.83e-04 | 4636.8ms/step | 13.8 samples/s
432
+ Step 3290/10005 | Loss: 3.4141 | LR: 3.83e-04 | 4636.5ms/step | 13.8 samples/s
433
+ Step 3300/10005 | Loss: 3.4277 | LR: 3.82e-04 | 4639.6ms/step | 13.8 samples/s
434
+ Step 3310/10005 | Loss: 3.4020 | LR: 3.81e-04 | 4636.6ms/step | 13.8 samples/s
435
+ Step 3320/10005 | Loss: 3.4205 | LR: 3.81e-04 | 4633.5ms/step | 13.8 samples/s
436
+ Step 3330/10005 | Loss: 3.4053 | LR: 3.80e-04 | 4630.0ms/step | 13.8 samples/s
437
+ Step 3340/10005 | Loss: 3.3991 | LR: 3.79e-04 | 4633.1ms/step | 13.8 samples/s
438
+ Step 3350/10005 | Loss: 3.4349 | LR: 3.79e-04 | 4630.8ms/step | 13.8 samples/s
439
+ Step 3360/10005 | Loss: 3.3922 | LR: 3.78e-04 | 4631.1ms/step | 13.8 samples/s
440
+ Step 3370/10005 | Loss: 3.3907 | LR: 3.77e-04 | 4625.9ms/step | 13.8 samples/s
441
+ Step 3380/10005 | Loss: 3.3961 | LR: 3.76e-04 | 4631.5ms/step | 13.8 samples/s
442
+ Step 3390/10005 | Loss: 3.3849 | LR: 3.76e-04 | 4632.3ms/step | 13.8 samples/s
443
+ Step 3400/10005 | Loss: 3.4176 | LR: 3.75e-04 | 4630.0ms/step | 13.8 samples/s
444
+ Step 3410/10005 | Loss: 3.4082 | LR: 3.74e-04 | 4632.5ms/step | 13.8 samples/s
445
+ Step 3420/10005 | Loss: 3.4027 | LR: 3.74e-04 | 4629.7ms/step | 13.8 samples/s
446
+ Step 3430/10005 | Loss: 3.3997 | LR: 3.73e-04 | 4627.0ms/step | 13.8 samples/s
447
+ Step 3440/10005 | Loss: 3.3883 | LR: 3.72e-04 | 4625.3ms/step | 13.8 samples/s
448
+ Step 3450/10005 | Loss: 3.4088 | LR: 3.72e-04 | 4628.4ms/step | 13.8 samples/s
449
+ Step 3460/10005 | Loss: 3.3819 | LR: 3.71e-04 | 4629.8ms/step | 13.8 samples/s
450
+ Step 3470/10005 | Loss: 3.3939 | LR: 3.70e-04 | 4628.0ms/step | 13.8 samples/s
451
+ Step 3480/10005 | Loss: 3.3784 | LR: 3.70e-04 | 4630.2ms/step | 13.8 samples/s
452
+ Step 3490/10005 | Loss: 3.3857 | LR: 3.69e-04 | 4722.6ms/step | 13.6 samples/s
453
+ Step 3500/10005 | Loss: 3.3922 | LR: 3.68e-04 | 4627.8ms/step | 13.8 samples/s
454
+ Validation at step 3500: Loss = 3.3925, Perplexity = 29.74
455
+ Step 3510/10005 | Loss: 3.3816 | LR: 3.67e-04 | 76557.7ms/step | 0.8 samples/s
456
+ Step 3520/10005 | Loss: 3.3742 | LR: 3.67e-04 | 4590.8ms/step | 13.9 samples/s
457
+ Step 3530/10005 | Loss: 3.3911 | LR: 3.66e-04 | 4590.9ms/step | 13.9 samples/s
458
+ Step 3540/10005 | Loss: 3.3990 | LR: 3.65e-04 | 4589.6ms/step | 13.9 samples/s
459
+ Step 3550/10005 | Loss: 3.3845 | LR: 3.65e-04 | 4594.6ms/step | 13.9 samples/s
460
+ Step 3560/10005 | Loss: 3.3756 | LR: 3.64e-04 | 4590.5ms/step | 13.9 samples/s
461
+ Step 3570/10005 | Loss: 3.3739 | LR: 3.63e-04 | 4590.2ms/step | 13.9 samples/s
462
+ Step 3580/10005 | Loss: 3.3910 | LR: 3.63e-04 | 4599.0ms/step | 13.9 samples/s
463
+ Step 3590/10005 | Loss: 3.3889 | LR: 3.62e-04 | 4601.1ms/step | 13.9 samples/s
464
+ Step 3600/10005 | Loss: 3.3667 | LR: 3.61e-04 | 4593.9ms/step | 13.9 samples/s
465
+ Step 3610/10005 | Loss: 3.3851 | LR: 3.60e-04 | 4596.9ms/step | 13.9 samples/s
466
+ Step 3620/10005 | Loss: 3.3936 | LR: 3.60e-04 | 4593.2ms/step | 13.9 samples/s
467
+ Step 3630/10005 | Loss: 3.3843 | LR: 3.59e-04 | 4591.3ms/step | 13.9 samples/s
468
+ Step 3640/10005 | Loss: 3.3918 | LR: 3.58e-04 | 4594.5ms/step | 13.9 samples/s
469
+ Step 3650/10005 | Loss: 3.3644 | LR: 3.58e-04 | 4596.0ms/step | 13.9 samples/s
470
+ Step 3660/10005 | Loss: 3.3744 | LR: 3.57e-04 | 4595.5ms/step | 13.9 samples/s
471
+ Step 3670/10005 | Loss: 3.3689 | LR: 3.56e-04 | 4592.5ms/step | 13.9 samples/s
472
+ Step 3680/10005 | Loss: 3.3742 | LR: 3.55e-04 | 4592.0ms/step | 13.9 samples/s
473
+ Step 3690/10005 | Loss: 3.3733 | LR: 3.55e-04 | 4592.7ms/step | 13.9 samples/s
474
+ Step 3700/10005 | Loss: 3.3719 | LR: 3.54e-04 | 4591.2ms/step | 13.9 samples/s
475
+ Step 3710/10005 | Loss: 3.3598 | LR: 3.53e-04 | 4590.6ms/step | 13.9 samples/s
476
+ Step 3720/10005 | Loss: 3.3489 | LR: 3.53e-04 | 4590.2ms/step | 13.9 samples/s
477
+ Step 3730/10005 | Loss: 3.3762 | LR: 3.52e-04 | 4593.3ms/step | 13.9 samples/s
478
+ Step 3740/10005 | Loss: 3.3570 | LR: 3.51e-04 | 4593.2ms/step | 13.9 samples/s
479
+ Step 3750/10005 | Loss: 3.3462 | LR: 3.50e-04 | 4591.0ms/step | 13.9 samples/s
480
+ Validation at step 3750: Loss = 3.3670, Perplexity = 28.99
481
+ Step 3760/10005 | Loss: 3.3549 | LR: 3.50e-04 | 76557.7ms/step | 0.8 samples/s
482
+ Step 3770/10005 | Loss: 3.3845 | LR: 3.49e-04 | 4603.9ms/step | 13.9 samples/s
483
+ Step 3780/10005 | Loss: 3.3872 | LR: 3.48e-04 | 4600.3ms/step | 13.9 samples/s
484
+ Step 3790/10005 | Loss: 3.3578 | LR: 3.47e-04 | 4599.9ms/step | 13.9 samples/s
485
+ Step 3800/10005 | Loss: 3.3475 | LR: 3.47e-04 | 4600.2ms/step | 13.9 samples/s
486
+ Step 3810/10005 | Loss: 3.3459 | LR: 3.46e-04 | 4604.8ms/step | 13.9 samples/s
487
+ Step 3820/10005 | Loss: 3.3777 | LR: 3.45e-04 | 4603.1ms/step | 13.9 samples/s
488
+ Step 3830/10005 | Loss: 3.3628 | LR: 3.45e-04 | 4601.5ms/step | 13.9 samples/s
489
+ Step 3840/10005 | Loss: 3.3471 | LR: 3.44e-04 | 4600.3ms/step | 13.9 samples/s
490
+ Step 3850/10005 | Loss: 3.3670 | LR: 3.43e-04 | 4604.8ms/step | 13.9 samples/s
491
+ Step 3860/10005 | Loss: 3.3437 | LR: 3.42e-04 | 4601.6ms/step | 13.9 samples/s
492
+ Step 3870/10005 | Loss: 3.3200 | LR: 3.42e-04 | 4601.4ms/step | 13.9 samples/s
493
+ Step 3880/10005 | Loss: 3.3646 | LR: 3.41e-04 | 4601.7ms/step | 13.9 samples/s
494
+ Step 3890/10005 | Loss: 3.3456 | LR: 3.40e-04 | 4598.3ms/step | 13.9 samples/s
495
+ Step 3900/10005 | Loss: 3.3302 | LR: 3.39e-04 | 4600.0ms/step | 13.9 samples/s
496
+ Step 3910/10005 | Loss: 3.3724 | LR: 3.39e-04 | 4602.5ms/step | 13.9 samples/s
497
+ Step 3920/10005 | Loss: 3.3389 | LR: 3.38e-04 | 4601.9ms/step | 13.9 samples/s
498
+ Step 3930/10005 | Loss: 3.3631 | LR: 3.37e-04 | 4600.4ms/step | 13.9 samples/s
499
+ Step 3940/10005 | Loss: 3.3440 | LR: 3.36e-04 | 4604.5ms/step | 13.9 samples/s
500
+ Step 3950/10005 | Loss: 3.3312 | LR: 3.36e-04 | 4602.4ms/step | 13.9 samples/s
501
+ Step 3960/10005 | Loss: 3.3537 | LR: 3.35e-04 | 4602.3ms/step | 13.9 samples/s
502
+ Step 3970/10005 | Loss: 3.3384 | LR: 3.34e-04 | 4674.6ms/step | 13.7 samples/s
503
+ Step 3980/10005 | Loss: 3.3600 | LR: 3.33e-04 | 4601.3ms/step | 13.9 samples/s
504
+ Step 3990/10005 | Loss: 3.3411 | LR: 3.33e-04 | 4607.7ms/step | 13.9 samples/s
505
+ Step 4000/10005 | Loss: 3.3410 | LR: 3.32e-04 | 4610.2ms/step | 13.9 samples/s
506
+ Validation at step 4000: Loss = 3.3452, Perplexity = 28.37
507
+ Step 4010/10005 | Loss: 3.3652 | LR: 3.31e-04 | 76515.1ms/step | 0.8 samples/s
508
+ Step 4020/10005 | Loss: 3.3579 | LR: 3.30e-04 | 4590.9ms/step | 13.9 samples/s
509
+ Step 4030/10005 | Loss: 3.3543 | LR: 3.30e-04 | 4590.3ms/step | 13.9 samples/s
510
+ Step 4040/10005 | Loss: 3.3342 | LR: 3.29e-04 | 4591.9ms/step | 13.9 samples/s
511
+ Step 4050/10005 | Loss: 3.3522 | LR: 3.28e-04 | 4588.9ms/step | 13.9 samples/s
512
+ Step 4060/10005 | Loss: 3.3421 | LR: 3.27e-04 | 4591.3ms/step | 13.9 samples/s
513
+ Step 4070/10005 | Loss: 3.3085 | LR: 3.27e-04 | 4589.7ms/step | 13.9 samples/s
514
+ Step 4080/10005 | Loss: 3.3302 | LR: 3.26e-04 | 4590.0ms/step | 13.9 samples/s
515
+ Step 4090/10005 | Loss: 3.3477 | LR: 3.25e-04 | 4588.8ms/step | 13.9 samples/s
516
+ Step 4100/10005 | Loss: 3.3541 | LR: 3.24e-04 | 4586.0ms/step | 14.0 samples/s
517
+ Step 4110/10005 | Loss: 3.3183 | LR: 3.24e-04 | 4594.9ms/step | 13.9 samples/s
518
+ Step 4120/10005 | Loss: 3.3225 | LR: 3.23e-04 | 4590.9ms/step | 13.9 samples/s
519
+ Step 4130/10005 | Loss: 3.3374 | LR: 3.22e-04 | 4588.9ms/step | 13.9 samples/s
520
+ Step 4140/10005 | Loss: 3.3640 | LR: 3.21e-04 | 4588.5ms/step | 13.9 samples/s
521
+ Step 4150/10005 | Loss: 3.3232 | LR: 3.21e-04 | 4591.9ms/step | 13.9 samples/s
522
+ Step 4160/10005 | Loss: 3.3348 | LR: 3.20e-04 | 4587.0ms/step | 14.0 samples/s
523
+ Step 4170/10005 | Loss: 3.3243 | LR: 3.19e-04 | 4590.3ms/step | 13.9 samples/s
524
+ Step 4180/10005 | Loss: 3.3193 | LR: 3.18e-04 | 4590.4ms/step | 13.9 samples/s
525
+ Step 4190/10005 | Loss: 3.3190 | LR: 3.18e-04 | 4588.9ms/step | 13.9 samples/s
526
+ Step 4200/10005 | Loss: 3.2930 | LR: 3.17e-04 | 4591.8ms/step | 13.9 samples/s
527
+ Step 4210/10005 | Loss: 3.3394 | LR: 3.16e-04 | 4588.4ms/step | 13.9 samples/s
528
+ Step 4220/10005 | Loss: 3.3347 | LR: 3.15e-04 | 4587.5ms/step | 14.0 samples/s
529
+ Step 4230/10005 | Loss: 3.3246 | LR: 3.14e-04 | 4589.9ms/step | 13.9 samples/s
530
+ Step 4240/10005 | Loss: 3.3234 | LR: 3.14e-04 | 4589.6ms/step | 13.9 samples/s
531
+ Step 4250/10005 | Loss: 3.3306 | LR: 3.13e-04 | 4593.7ms/step | 13.9 samples/s
532
+ Validation at step 4250: Loss = 3.3234, Perplexity = 27.75
533
+ Step 4260/10005 | Loss: 3.3005 | LR: 3.12e-04 | 76560.0ms/step | 0.8 samples/s
534
+ Step 4270/10005 | Loss: 3.3352 | LR: 3.11e-04 | 4621.0ms/step | 13.8 samples/s
535
+ Step 4280/10005 | Loss: 3.3166 | LR: 3.11e-04 | 4620.9ms/step | 13.9 samples/s
536
+ Step 4290/10005 | Loss: 3.2975 | LR: 3.10e-04 | 4614.0ms/step | 13.9 samples/s
537
+ Step 4300/10005 | Loss: 3.3349 | LR: 3.09e-04 | 4627.2ms/step | 13.8 samples/s
538
+ Step 4310/10005 | Loss: 3.3163 | LR: 3.08e-04 | 4609.6ms/step | 13.9 samples/s
539
+ Step 4320/10005 | Loss: 3.3091 | LR: 3.08e-04 | 4622.6ms/step | 13.8 samples/s
540
+ Step 4330/10005 | Loss: 3.3123 | LR: 3.07e-04 | 4620.7ms/step | 13.9 samples/s
541
+ Step 4340/10005 | Loss: 3.3348 | LR: 3.06e-04 | 4616.8ms/step | 13.9 samples/s
542
+ Step 4350/10005 | Loss: 3.3238 | LR: 3.05e-04 | 4624.8ms/step | 13.8 samples/s
543
+ Step 4360/10005 | Loss: 3.3263 | LR: 3.04e-04 | 4628.1ms/step | 13.8 samples/s
544
+ Step 4370/10005 | Loss: 3.3166 | LR: 3.04e-04 | 4624.1ms/step | 13.8 samples/s
545
+ Step 4380/10005 | Loss: 3.3395 | LR: 3.03e-04 | 4622.6ms/step | 13.8 samples/s
546
+ Step 4390/10005 | Loss: 3.2834 | LR: 3.02e-04 | 4625.1ms/step | 13.8 samples/s
547
+ Step 4400/10005 | Loss: 3.2805 | LR: 3.01e-04 | 4623.9ms/step | 13.8 samples/s
548
+ Step 4410/10005 | Loss: 3.3148 | LR: 3.01e-04 | 4629.6ms/step | 13.8 samples/s
549
+ Step 4420/10005 | Loss: 3.3170 | LR: 3.00e-04 | 4628.7ms/step | 13.8 samples/s
550
+ Step 4430/10005 | Loss: 3.3039 | LR: 2.99e-04 | 4626.4ms/step | 13.8 samples/s
551
+ Step 4440/10005 | Loss: 3.3083 | LR: 2.98e-04 | 4683.3ms/step | 13.7 samples/s
552
+ Step 4450/10005 | Loss: 3.2999 | LR: 2.97e-04 | 4622.6ms/step | 13.8 samples/s
553
+ Step 4460/10005 | Loss: 3.3138 | LR: 2.97e-04 | 4622.1ms/step | 13.8 samples/s
554
+ Step 4470/10005 | Loss: 3.3163 | LR: 2.96e-04 | 4625.0ms/step | 13.8 samples/s
555
+ Step 4480/10005 | Loss: 3.2941 | LR: 2.95e-04 | 4623.6ms/step | 13.8 samples/s
556
+ Step 4490/10005 | Loss: 3.2869 | LR: 2.94e-04 | 4622.5ms/step | 13.8 samples/s
557
+ Step 4500/10005 | Loss: 3.3031 | LR: 2.94e-04 | 4624.3ms/step | 13.8 samples/s
558
+ Validation at step 4500: Loss = 3.3039, Perplexity = 27.22
559
+ Step 4510/10005 | Loss: 3.2922 | LR: 2.93e-04 | 76702.6ms/step | 0.8 samples/s
560
+ Step 4520/10005 | Loss: 3.3224 | LR: 2.92e-04 | 4623.6ms/step | 13.8 samples/s
561
+ Step 4530/10005 | Loss: 3.2966 | LR: 2.91e-04 | 4617.9ms/step | 13.9 samples/s
562
+ Step 4540/10005 | Loss: 3.2931 | LR: 2.90e-04 | 4624.5ms/step | 13.8 samples/s
563
+ Step 4550/10005 | Loss: 3.2915 | LR: 2.90e-04 | 4620.3ms/step | 13.9 samples/s
564
+ Step 4560/10005 | Loss: 3.3020 | LR: 2.89e-04 | 4618.8ms/step | 13.9 samples/s
565
+ Step 4570/10005 | Loss: 3.2919 | LR: 2.88e-04 | 4620.0ms/step | 13.9 samples/s
566
+ Step 4580/10005 | Loss: 3.3037 | LR: 2.87e-04 | 4620.5ms/step | 13.9 samples/s
567
+ Step 4590/10005 | Loss: 3.2826 | LR: 2.87e-04 | 4620.9ms/step | 13.9 samples/s
568
+ Step 4600/10005 | Loss: 3.2662 | LR: 2.86e-04 | 4619.6ms/step | 13.9 samples/s
569
+ Step 4610/10005 | Loss: 3.3105 | LR: 2.85e-04 | 4618.7ms/step | 13.9 samples/s
570
+ Step 4620/10005 | Loss: 3.3022 | LR: 2.84e-04 | 4619.9ms/step | 13.9 samples/s
571
+ Step 4630/10005 | Loss: 3.2964 | LR: 2.83e-04 | 4616.5ms/step | 13.9 samples/s
572
+ Step 4640/10005 | Loss: 3.3234 | LR: 2.83e-04 | 4618.5ms/step | 13.9 samples/s
573
+ Step 4650/10005 | Loss: 3.2832 | LR: 2.82e-04 | 4617.7ms/step | 13.9 samples/s
574
+ Step 4660/10005 | Loss: 3.3000 | LR: 2.81e-04 | 4618.2ms/step | 13.9 samples/s
575
+ Step 4670/10005 | Loss: 3.3045 | LR: 2.80e-04 | 4619.7ms/step | 13.9 samples/s
576
+ Step 4680/10005 | Loss: 3.2898 | LR: 2.79e-04 | 4621.6ms/step | 13.8 samples/s
577
+ Step 4690/10005 | Loss: 3.2916 | LR: 2.79e-04 | 4617.3ms/step | 13.9 samples/s
578
+ Step 4700/10005 | Loss: 3.2817 | LR: 2.78e-04 | 4620.4ms/step | 13.9 samples/s
579
+ Step 4710/10005 | Loss: 3.3110 | LR: 2.77e-04 | 4619.2ms/step | 13.9 samples/s
580
+ Step 4720/10005 | Loss: 3.2792 | LR: 2.76e-04 | 4619.0ms/step | 13.9 samples/s
581
+ Step 4730/10005 | Loss: 3.2788 | LR: 2.76e-04 | 4620.4ms/step | 13.9 samples/s
582
+ Step 4740/10005 | Loss: 3.3007 | LR: 2.75e-04 | 4618.6ms/step | 13.9 samples/s
583
+ Step 4750/10005 | Loss: 3.2788 | LR: 2.74e-04 | 4618.1ms/step | 13.9 samples/s
584
+ Validation at step 4750: Loss = 3.2858, Perplexity = 26.73
585
+ Step 4760/10005 | Loss: 3.3021 | LR: 2.73e-04 | 76631.9ms/step | 0.8 samples/s
586
+ Step 4770/10005 | Loss: 3.3009 | LR: 2.72e-04 | 4622.1ms/step | 13.8 samples/s
587
+ Step 4780/10005 | Loss: 3.2691 | LR: 2.72e-04 | 4618.3ms/step | 13.9 samples/s
588
+ Step 4790/10005 | Loss: 3.3022 | LR: 2.71e-04 | 4618.5ms/step | 13.9 samples/s
589
+ Step 4800/10005 | Loss: 3.2797 | LR: 2.70e-04 | 4618.2ms/step | 13.9 samples/s
590
+ Step 4810/10005 | Loss: 3.2937 | LR: 2.69e-04 | 4617.8ms/step | 13.9 samples/s
591
+ Step 4820/10005 | Loss: 3.2696 | LR: 2.68e-04 | 4617.4ms/step | 13.9 samples/s
592
+ Step 4830/10005 | Loss: 3.2755 | LR: 2.68e-04 | 4620.2ms/step | 13.9 samples/s
593
+ Step 4840/10005 | Loss: 3.2905 | LR: 2.67e-04 | 4617.2ms/step | 13.9 samples/s
594
+ Step 4850/10005 | Loss: 3.2640 | LR: 2.66e-04 | 4616.5ms/step | 13.9 samples/s
595
+ Step 4860/10005 | Loss: 3.2662 | LR: 2.65e-04 | 4623.3ms/step | 13.8 samples/s
596
+ Step 4870/10005 | Loss: 3.2671 | LR: 2.64e-04 | 4620.5ms/step | 13.9 samples/s
597
+ Step 4880/10005 | Loss: 3.2663 | LR: 2.64e-04 | 4616.3ms/step | 13.9 samples/s
598
+ Saved checkpoint to checkpoints/run_20260430_154424/checkpoint-4882
epfl_final/wandb/run-20260430_154427-zytgzzs7/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.1.post1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.4.3
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.58.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.3
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.3
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.49
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.0
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.13.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.7.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260430_154427-zytgzzs7/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-04-30T15:44:27.541055Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-0-6",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6053369475072"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-fbf0ccd1-8bd4-5f57-6e74-6c0cf7eb70c9"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "3utjvuetabvi926o8bezxiyicqfp65jq"
36
+ }
epfl_final/wandb/run-20260430_154427-zytgzzs7/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"speed/samples_per_sec":13.864066967903103,"train/global_step":4880,"train/epoch":0,"_wandb":{"runtime":36298},"_timestamp":1.777600151786837e+09,"eval/perplexity":26.73106351599795,"_runtime":36298.349780915,"speed/ms_per_step":4616.250061988831,"_step":4880,"train/loss":3.2663412392139435,"speed/steps_per_sec":0.21662604637348598,"train/learning_rate":0.0002636712368582849,"eval/loss":3.285826316714287}
epfl_final/wandb/run-20260430_154427-zytgzzs7/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/wandb/run-20260430_154427-zytgzzs7/logs/debug.log ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_setup.py:_flush():81] Configure stats pid to 224
3
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_154427-zytgzzs7/logs/debug.log
5
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_154427-zytgzzs7/logs/debug-internal.log
6
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_init.py:init():850] calling init triggers
7
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 3840, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_154424', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10005, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 9, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-04-30 15:44:27,558 INFO MainThread:224 [wandb_init.py:init():898] starting backend
10
+ 2026-04-30 15:44:27,786 INFO MainThread:224 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-04-30 15:44:28,087 INFO MainThread:224 [wandb_init.py:init():918] backend started and connected
12
+ 2026-04-30 15:44:28,090 INFO MainThread:224 [wandb_init.py:init():988] updated telemetry
13
+ 2026-04-30 15:44:28,091 INFO MainThread:224 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-04-30 15:44:28,436 INFO MainThread:224 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-04-30 15:44:28,523 INFO MainThread:224 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-04-30 15:44:28,523 INFO MainThread:224 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-04-30 15:44:28,523 INFO MainThread:224 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-04-30 15:44:28,523 INFO MainThread:224 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-04-30 15:44:28,527 INFO MainThread:224 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-05-01 01:49:26,784 INFO MainThread:224 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/zytgzzs7
21
+ 2026-05-01 01:49:26,785 INFO MainThread:224 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
22
+ 2026-05-01 01:49:26,787 INFO MainThread:224 [wandb_run.py:_restore():2502] restore
23
+ 2026-05-01 01:49:26,787 INFO MainThread:224 [wandb_run.py:_restore():2508] restore done
24
+ 2026-05-01 01:49:27,871 INFO MainThread:224 [wandb_run.py:_footer_sync_info():3889] logging synced files
epfl_final/wandb/run-20260430_154427-zytgzzs7/run-zytgzzs7.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89e5a17bb0885ca37d68d707565afa03e010c0b61025091f43e8c73a0377e388
3
+ size 5248325
epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/config.yaml ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ _wandb:
2
+ value:
3
+ cli_version: 0.26.1
4
+ e:
5
+ yvtqlqnl20kgitubkju4w1et8qu1lhrp:
6
+ codePath: SparseAtt/train.py
7
+ codePathLocal: SparseAtt/train.py
8
+ cpu_count: 96
9
+ cpu_count_logical: 192
10
+ cudaVersion: "13.0"
11
+ disk:
12
+ /:
13
+ total: "9374931091456"
14
+ used: "6049938808832"
15
+ email: ruben.navasardyan@mail.utoronto.ca
16
+ executable: /usr/bin/python3
17
+ gpu: NVIDIA H100 80GB HBM3
18
+ gpu_count: 1
19
+ gpu_nvidia:
20
+ - architecture: Hopper
21
+ cudaCores: 16896
22
+ memoryTotal: "85520809984"
23
+ name: NVIDIA H100 80GB HBM3
24
+ uuid: GPU-9e3de42d-518e-8887-4b3d-6c8316629c5e
25
+ host: h100-workspace-2-0-6
26
+ memory:
27
+ total: "1623005847552"
28
+ os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
29
+ program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
30
+ python: CPython 3.10.12
31
+ root: /mloscratch/homes/navasard/sparse
32
+ startedAt: "2026-04-30T15:45:31.889894Z"
33
+ writerId: yvtqlqnl20kgitubkju4w1et8qu1lhrp
34
+ m: []
35
+ python_version: 3.10.12
36
+ t:
37
+ "1":
38
+ - 1
39
+ - 49
40
+ - 51
41
+ "2":
42
+ - 1
43
+ - 11
44
+ - 49
45
+ - 51
46
+ - 71
47
+ "3":
48
+ - 15
49
+ - 16
50
+ - 61
51
+ "4": 3.10.12
52
+ "5": 0.26.1
53
+ "12": 0.26.1
54
+ "13": linux-x86_64
55
+ attention_type:
56
+ value: sparse_stochastic
57
+ checkpoint_activations:
58
+ value: false
59
+ eval_batches:
60
+ value: 500
61
+ eval_steps:
62
+ value: 250
63
+ gradient_accumulation_steps:
64
+ value: 8
65
+ init_from_scratch:
66
+ value: true
67
+ learning_rate:
68
+ value: 0.0005
69
+ logging_steps:
70
+ value: 10
71
+ lr_scheduler:
72
+ value: cosine
73
+ max_grad_norm:
74
+ value: 1
75
+ max_seq_length:
76
+ value: 3840
77
+ max_steps:
78
+ value: 10005
79
+ mixed_precision:
80
+ value: bf16
81
+ model_dtype:
82
+ value: bfloat16
83
+ model_path:
84
+ value: /mloscratch/homes/navasard/qwen3_06b_base
85
+ num_epochs:
86
+ value: 1
87
+ num_workers:
88
+ value: 8
89
+ output_dir:
90
+ value: ./checkpoints/run_20260430_154528
91
+ per_device_batch_size:
92
+ value: 8
93
+ resume_from_checkpoint:
94
+ value: null
95
+ save_steps:
96
+ value: 2500
97
+ seed:
98
+ value: 42
99
+ sparse_block_size:
100
+ value: 128
101
+ sparse_layer_indices:
102
+ value: null
103
+ sparse_mask_pool_size:
104
+ value: 1000
105
+ sparse_num_random_blocks:
106
+ value: 14
107
+ sparse_val_dense_compare:
108
+ value: true
109
+ strategy:
110
+ value: ddp
111
+ train_path:
112
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
113
+ val_path:
114
+ value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
115
+ verbose:
116
+ value: true
117
+ wandb_enabled:
118
+ value: true
119
+ wandb_name:
120
+ value: null
121
+ wandb_project:
122
+ value: stochastic_sparsity
123
+ wandb_tags:
124
+ value:
125
+ - qwen3
126
+ - 0.6b
127
+ warmup_steps:
128
+ value: 100
129
+ weight_decay:
130
+ value: 0.01
epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/output.log ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
2
+ Initializing model from scratch (random weights)...
3
+ Checkpoint activations: False
4
+
5
+
6
+ ================================================================================
7
+ MODEL ARCHITECTURE
8
+ ================================================================================
9
+ Model config:
10
+ vocab_size: 151936
11
+ hidden_size: 1024
12
+ num_hidden_layers: 28
13
+ num_attention_heads: 16
14
+ num_key_value_heads: 8
15
+ head_dim: 128
16
+ intermediate_size: 3072
17
+ max_position_embeddings: 32768
18
+ rope_theta: 1000000
19
+ attention_type: sparse_stochastic
20
+ sparse_block_size: 128
21
+ sparse_num_random_blocks: 14
22
+ sparse_mask_pool_size: 1000
23
+ sparse_max_seq_length: 3840
24
+
25
+ Parameter count:
26
+ Total parameters: 596,049,920
27
+ Trainable parameters: 596,049,920
28
+ Non-trainable parameters: 0
29
+ Model size (BF16): 1.19 GB
30
+
31
+ --------------------------------------------------------------------------------
32
+ LAYER-BY-LAYER SUMMARY
33
+ --------------------------------------------------------------------------------
34
+ /home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
35
+
36
+ SOLUTION: Use torch.compile(flex_attention)(...)
37
+
38
+ If you want to debug your score_mod/mask_mod, you can set:
39
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
40
+
41
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
42
+ _warn_once(
43
+ ================================================================================================================================================================
44
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
45
+ ================================================================================================================================================================
46
+ Qwen3ForCausalLM [8, 3840] -- -- True
47
+ ├─Qwen3Model: 1-1 -- [8, 3840, 1024] -- True
48
+ │ └─Embedding: 2-1 [8, 3840] [8, 3840, 1024] 155,582,464 True
49
+ │ └─ModuleList: 2-2 -- -- -- True
50
+ │ │ └─Qwen3DecoderLayer: 3-1 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-2 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-3 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-4 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-5 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-6 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-7 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-8 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-9 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-10 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-11 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-12 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-13 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-14 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-15 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-16 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-17 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-18 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-19 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-20 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-21 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-22 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-23 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-24 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-25 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
75
+ │ │ └─Qwen3DecoderLayer: 3-26 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
76
+ │ │ └─Qwen3DecoderLayer: 3-27 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
77
+ │ │ └─Qwen3DecoderLayer: 3-28 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
78
+ │ └─Qwen3RMSNorm: 2-3 [8, 3840, 1024] [8, 3840, 1024] 1,024 True
79
+ ├─Linear: 1-2 [8, 3840, 1024] [8, 3840, 151936] 155,582,464 True
80
+ ================================================================================================================================================================
81
+ Total params: 751,632,384
82
+ Trainable params: 751,632,384
83
+ Non-trainable params: 0
84
+ Total mult-adds (G): 6.01
85
+ ================================================================================================================================================================
86
+ Input size (MB): 0.25
87
+ Forward/backward pass size (MB): 78816.22
88
+ Params size (MB): 1503.26
89
+ Estimated Total Size (MB): 80319.73
90
+ ================================================================================================================================================================
91
+ ================================================================================
92
+
93
+ Checkpoint activations: False
94
+ Sparse validation compare enabled: running both sparse and dense validation passes
95
+ Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
96
+ Using pre-tokenized dataset (no runtime tokenization)
97
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
98
+ Loaded 781148 pre-tokenized examples (stored_len=2048, pack_factor=2, packed_examples=390574)
99
+ Dataset split: 312459 train, 78115 validation
100
+ Training examples: 312459
101
+ Validation examples: 78115
102
+ Steps per epoch: 4882
103
+ Total training steps: 10005
104
+ Starting training...
105
+ Step 10/10005 | Loss: 11.8633 | LR: 5.00e-05 | 10195.9ms/step | 6.3 samples/s
106
+ Step 20/10005 | Loss: 10.6535 | LR: 1.00e-04 | 2798.6ms/step | 22.9 samples/s
107
+ Step 30/10005 | Loss: 9.6631 | LR: 1.50e-04 | 2794.3ms/step | 22.9 samples/s
108
+ Step 40/10005 | Loss: 8.5237 | LR: 2.00e-04 | 2796.6ms/step | 22.9 samples/s
109
+ Step 50/10005 | Loss: 8.0146 | LR: 2.50e-04 | 2961.0ms/step | 21.6 samples/s
110
+ Step 60/10005 | Loss: 7.8453 | LR: 3.00e-04 | 2809.2ms/step | 22.8 samples/s
111
+ Step 70/10005 | Loss: 7.6546 | LR: 3.50e-04 | 2814.0ms/step | 22.7 samples/s
112
+ Step 80/10005 | Loss: 7.4184 | LR: 4.00e-04 | 2815.0ms/step | 22.7 samples/s
113
+ Step 90/10005 | Loss: 7.2064 | LR: 4.50e-04 | 2820.0ms/step | 22.7 samples/s
114
+ Step 100/10005 | Loss: 7.0387 | LR: 5.00e-04 | 2815.1ms/step | 22.7 samples/s
115
+ Step 110/10005 | Loss: 6.9117 | LR: 5.00e-04 | 2816.7ms/step | 22.7 samples/s
116
+ Step 120/10005 | Loss: 6.7916 | LR: 5.00e-04 | 2820.1ms/step | 22.7 samples/s
117
+ Step 130/10005 | Loss: 6.7169 | LR: 5.00e-04 | 2817.5ms/step | 22.7 samples/s
118
+ Step 140/10005 | Loss: 6.6459 | LR: 5.00e-04 | 2821.5ms/step | 22.7 samples/s
119
+ Step 150/10005 | Loss: 6.5337 | LR: 5.00e-04 | 2821.8ms/step | 22.7 samples/s
120
+ Step 160/10005 | Loss: 6.4470 | LR: 5.00e-04 | 2820.1ms/step | 22.7 samples/s
121
+ Step 170/10005 | Loss: 6.3909 | LR: 5.00e-04 | 2822.0ms/step | 22.7 samples/s
122
+ Step 180/10005 | Loss: 6.3392 | LR: 5.00e-04 | 2822.7ms/step | 22.7 samples/s
123
+ Step 190/10005 | Loss: 6.2785 | LR: 5.00e-04 | 2820.6ms/step | 22.7 samples/s
124
+ Step 200/10005 | Loss: 6.1569 | LR: 5.00e-04 | 2820.0ms/step | 22.7 samples/s
125
+ Step 210/10005 | Loss: 6.1388 | LR: 5.00e-04 | 2823.0ms/step | 22.7 samples/s
126
+ Step 220/10005 | Loss: 6.1240 | LR: 5.00e-04 | 2821.6ms/step | 22.7 samples/s
127
+ Step 230/10005 | Loss: 6.0028 | LR: 5.00e-04 | 2826.1ms/step | 22.6 samples/s
128
+ Step 240/10005 | Loss: 5.9292 | LR: 5.00e-04 | 2822.6ms/step | 22.7 samples/s
129
+ Step 250/10005 | Loss: 5.9298 | LR: 5.00e-04 | 2820.5ms/step | 22.7 samples/s
130
+ Validation at step 250: Loss = 5.8845, Perplexity = 359.43
131
+ Dense validation compare at step 250: Loss = 5.8849, Perplexity = 359.55
132
+ Traceback (most recent call last):
133
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1069, in <module>
134
+ main()
135
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1065, in main
136
+ train(config)
137
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 926, in train
138
+ loss.backward()
139
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_tensor.py", line 631, in backward
140
+ torch.autograd.backward(
141
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/__init__.py", line 381, in backward
142
+ _engine_run_backward(
143
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
144
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
145
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 317, in apply
146
+ return user_fn(self, *args)
147
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2748, in backward
148
+ return impl_fn()
149
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2734, in impl_fn
150
+ out = CompiledFunction._backward_impl(ctx, all_args)
151
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2918, in _backward_impl
152
+ out = call_func_at_runtime_with_args(
153
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 138, in call_func_at_runtime_with_args
154
+ out = normalize_as_list(f(args))
155
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
156
+ return fn(*args, **kwargs)
157
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 656, in __call__
158
+ return self.current_callable(inputs)
159
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3401, in run
160
+ out = model(new_inputs)
161
+ File "/tmp/torchinductor_navasard/vo/cvothtmgm6kahrnwz6es5wzrb2b765nmhrv6ponmr3rjxen2e3ki.py", line 305, in call
162
+ buf1 = empty_strided_cuda((8, 3840, 151936), (583434240, 151936, 1), torch.bfloat16)
163
+ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 8.70 GiB. GPU 0 has a total capacity of 79.18 GiB of which 2.63 GiB is free. Including non-PyTorch memory, this process has 76.51 GiB memory in use. Of the allocated memory 62.23 GiB is allocated by PyTorch, and 13.55 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
164
+ Traceback (most recent call last):
165
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1069, in <module>
166
+ main()
167
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1065, in main
168
+ train(config)
169
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 926, in train
170
+ loss.backward()
171
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_tensor.py", line 631, in backward
172
+ torch.autograd.backward(
173
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/__init__.py", line 381, in backward
174
+ _engine_run_backward(
175
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
176
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
177
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 317, in apply
178
+ return user_fn(self, *args)
179
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2748, in backward
180
+ return impl_fn()
181
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2734, in impl_fn
182
+ out = CompiledFunction._backward_impl(ctx, all_args)
183
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2918, in _backward_impl
184
+ out = call_func_at_runtime_with_args(
185
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 138, in call_func_at_runtime_with_args
186
+ out = normalize_as_list(f(args))
187
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
188
+ return fn(*args, **kwargs)
189
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 656, in __call__
190
+ return self.current_callable(inputs)
191
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3401, in run
192
+ out = model(new_inputs)
193
+ File "/tmp/torchinductor_navasard/vo/cvothtmgm6kahrnwz6es5wzrb2b765nmhrv6ponmr3rjxen2e3ki.py", line 305, in call
194
+ buf1 = empty_strided_cuda((8, 3840, 151936), (583434240, 151936, 1), torch.bfloat16)
195
+ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 8.70 GiB. GPU 0 has a total capacity of 79.18 GiB of which 2.63 GiB is free. Including non-PyTorch memory, this process has 76.51 GiB memory in use. Of the allocated memory 62.23 GiB is allocated by PyTorch, and 13.55 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.1.post1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.4.3
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.58.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.3
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.3
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.49
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.0
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.13.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.7.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-04-30T15:45:31.889894Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-2-0-6",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6049938808832"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-9e3de42d-518e-8887-4b3d-6c8316629c5e"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "yvtqlqnl20kgitubkju4w1et8qu1lhrp"
36
+ }
epfl_final/wandb/run-20260430_154531-8xz0mg7d/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"eval/loss":5.884514079928398,"_runtime":3353.226075827,"train/epoch":0,"speed/steps_per_sec":0.3545489731893506,"_timestamp":1.77756720518832e+09,"eval/perplexity":359.428072241356,"_step":250,"eval_dense_compare/perplexity":359.54996665408896,"train/learning_rate":0.0004997171205507588,"_wandb":{"runtime":3353},"speed/ms_per_step":2820.4848289489746,"train/loss":5.929800093173981,"train/global_step":250,"eval_dense_compare/loss":5.884853156805039,"speed/samples_per_sec":22.691134284118437}
epfl_final/wandb/run-20260430_154531-8xz0mg7d/logs/debug-internal.log ADDED
@@ -0,0 +1,462 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-04-30T15:45:32.143762106Z","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-04-30T15:45:32.144081294Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
3
+ {"time":"2026-04-30T15:45:32.440112652Z","level":"INFO","msg":"stream: created new stream","id":"8xz0mg7d"}
4
+ {"time":"2026-04-30T15:45:32.440221876Z","level":"INFO","msg":"handler: started"}
5
+ {"time":"2026-04-30T15:45:32.442348313Z","level":"INFO","msg":"stream: started"}
6
+ {"time":"2026-04-30T15:45:32.442369335Z","level":"INFO","msg":"writer: started","stream_id":"8xz0mg7d"}
7
+ {"time":"2026-04-30T15:45:32.44239332Z","level":"INFO","msg":"sender: started"}
8
+ {"time":"2026-04-30T15:45:33.005227903Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
9
+ {"time":"2026-04-30T15:45:33.261857559Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-04-30T15:45:48.005930294Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":2,"uploaded_len":2}
11
+ {"time":"2026-04-30T15:45:48.290543877Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-04-30T15:46:03.005565305Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
13
+ {"time":"2026-04-30T15:46:03.331812596Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
14
+ {"time":"2026-04-30T15:46:18.00539037Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":3,"events_lines":2}
15
+ {"time":"2026-04-30T15:46:18.330981898Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
16
+ {"time":"2026-04-30T15:46:33.005674744Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
17
+ {"time":"2026-04-30T15:46:33.256335612Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
18
+ {"time":"2026-04-30T15:46:48.00622708Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":7,"events_lines":2}
19
+ {"time":"2026-04-30T15:46:48.361745849Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
20
+ {"time":"2026-04-30T15:47:03.005599046Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":9,"events_lines":2}
21
+ {"time":"2026-04-30T15:47:03.382550583Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
22
+ {"time":"2026-04-30T15:47:18.00529605Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":11,"events_lines":2}
23
+ {"time":"2026-04-30T15:47:18.312544983Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
24
+ {"time":"2026-04-30T15:47:33.006137509Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":13,"events_lines":2}
25
+ {"time":"2026-04-30T15:47:33.255196486Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
26
+ {"time":"2026-04-30T15:47:48.005427442Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":15,"events_lines":2}
27
+ {"time":"2026-04-30T15:47:48.261489747Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
28
+ {"time":"2026-04-30T15:48:03.005646689Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":17,"events_lines":2}
29
+ {"time":"2026-04-30T15:48:03.309567232Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
30
+ {"time":"2026-04-30T15:48:18.005899105Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":19,"events_lines":2}
31
+ {"time":"2026-04-30T15:48:18.314451441Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
32
+ {"time":"2026-04-30T15:48:33.005982898Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":21,"events_lines":2}
33
+ {"time":"2026-04-30T15:48:33.262571081Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
34
+ {"time":"2026-04-30T15:48:48.005638322Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":23,"events_lines":2}
35
+ {"time":"2026-04-30T15:48:48.290501014Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
36
+ {"time":"2026-04-30T15:49:03.005709536Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":25,"events_lines":2}
37
+ {"time":"2026-04-30T15:49:03.315822798Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
38
+ {"time":"2026-04-30T15:49:18.005334831Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":27,"events_lines":2}
39
+ {"time":"2026-04-30T15:49:18.364039593Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
40
+ {"time":"2026-04-30T15:49:33.006160777Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":29,"events_lines":2}
41
+ {"time":"2026-04-30T15:49:33.254490265Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
42
+ {"time":"2026-04-30T15:49:48.0059514Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":31,"events_lines":2}
43
+ {"time":"2026-04-30T15:49:48.274664112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
44
+ {"time":"2026-04-30T15:50:03.00584034Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":33,"events_lines":2}
45
+ {"time":"2026-04-30T15:50:03.256523576Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
46
+ {"time":"2026-04-30T15:50:18.005963812Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":35,"events_lines":2}
47
+ {"time":"2026-04-30T15:50:18.572426895Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
48
+ {"time":"2026-04-30T15:50:33.005898301Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":37,"events_lines":2}
49
+ {"time":"2026-04-30T15:50:33.475666517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
50
+ {"time":"2026-04-30T15:50:48.006012229Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":39,"events_lines":2}
51
+ {"time":"2026-04-30T15:50:48.363744883Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
52
+ {"time":"2026-04-30T15:51:03.005563083Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":41,"events_lines":2}
53
+ {"time":"2026-04-30T15:51:03.310842068Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
54
+ {"time":"2026-04-30T15:51:18.005879183Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":43,"events_lines":2}
55
+ {"time":"2026-04-30T15:51:18.460513073Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
56
+ {"time":"2026-04-30T15:51:33.005227531Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":45,"events_lines":2}
57
+ {"time":"2026-04-30T15:51:33.508916695Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
58
+ {"time":"2026-04-30T15:51:48.005507808Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":47,"events_lines":2}
59
+ {"time":"2026-04-30T15:51:48.280087174Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
60
+ {"time":"2026-04-30T15:52:03.005845932Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":49,"events_lines":2}
61
+ {"time":"2026-04-30T15:52:03.362125217Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
62
+ {"time":"2026-04-30T15:52:18.00622678Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":51,"events_lines":2}
63
+ {"time":"2026-04-30T15:52:18.279267139Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
64
+ {"time":"2026-04-30T15:52:33.005954659Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":53,"events_lines":2}
65
+ {"time":"2026-04-30T15:52:33.257053257Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
66
+ {"time":"2026-04-30T15:52:48.006236098Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":55,"events_lines":2}
67
+ {"time":"2026-04-30T15:52:48.252592057Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
68
+ {"time":"2026-04-30T15:53:03.006232589Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":57,"events_lines":2}
69
+ {"time":"2026-04-30T15:53:03.357075045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
70
+ {"time":"2026-04-30T15:53:18.005762562Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":59,"events_lines":2}
71
+ {"time":"2026-04-30T15:53:18.285114573Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
72
+ {"time":"2026-04-30T15:53:33.00556311Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":61,"events_lines":2}
73
+ {"time":"2026-04-30T15:53:33.232071027Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
74
+ {"time":"2026-04-30T15:53:48.006207423Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":63,"events_lines":2}
75
+ {"time":"2026-04-30T15:53:48.389678359Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
76
+ {"time":"2026-04-30T15:54:03.00594065Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":65,"events_lines":2}
77
+ {"time":"2026-04-30T15:54:03.253854865Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
78
+ {"time":"2026-04-30T15:54:18.005885955Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":67,"events_lines":2}
79
+ {"time":"2026-04-30T15:54:18.292946731Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
80
+ {"time":"2026-04-30T15:54:33.00545769Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":69,"events_lines":2}
81
+ {"time":"2026-04-30T15:54:33.262212207Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
82
+ {"time":"2026-04-30T15:54:48.005803475Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":71,"events_lines":2}
83
+ {"time":"2026-04-30T15:54:48.28735308Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
84
+ {"time":"2026-04-30T15:55:03.006604233Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":73,"events_lines":2}
85
+ {"time":"2026-04-30T15:55:03.540407418Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
86
+ {"time":"2026-04-30T15:55:18.005395378Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":75,"events_lines":2}
87
+ {"time":"2026-04-30T15:55:18.277723159Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
88
+ {"time":"2026-04-30T15:55:33.005674323Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":77,"events_lines":2}
89
+ {"time":"2026-04-30T15:55:33.238378038Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
90
+ {"time":"2026-04-30T15:55:48.005424435Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":79,"events_lines":2}
91
+ {"time":"2026-04-30T15:55:48.271753855Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
92
+ {"time":"2026-04-30T15:56:03.005519695Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":81,"events_lines":2}
93
+ {"time":"2026-04-30T15:56:03.307058865Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
94
+ {"time":"2026-04-30T15:56:18.005262547Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":83,"events_lines":2}
95
+ {"time":"2026-04-30T15:56:18.307800553Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
96
+ {"time":"2026-04-30T15:56:33.006129172Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":85,"events_lines":2}
97
+ {"time":"2026-04-30T15:56:33.279217574Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
98
+ {"time":"2026-04-30T15:56:48.00594956Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":87,"events_lines":2}
99
+ {"time":"2026-04-30T15:56:48.341108787Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
100
+ {"time":"2026-04-30T15:57:03.005692272Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":89,"events_lines":2}
101
+ {"time":"2026-04-30T15:57:03.244239496Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
102
+ {"time":"2026-04-30T15:57:18.006190415Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":91,"events_lines":2}
103
+ {"time":"2026-04-30T15:57:18.382528639Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
104
+ {"time":"2026-04-30T15:57:33.00599542Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":93,"events_lines":2}
105
+ {"time":"2026-04-30T15:57:33.260514797Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
106
+ {"time":"2026-04-30T15:57:48.006578802Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":95,"events_lines":2}
107
+ {"time":"2026-04-30T15:57:48.26461472Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
108
+ {"time":"2026-04-30T15:58:03.006143666Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":97,"events_lines":2,"console_offset":3,"console_lines":39}
109
+ {"time":"2026-04-30T15:58:03.267113732Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
110
+ {"time":"2026-04-30T15:58:18.005888761Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":99,"events_lines":2,"console_offset":42,"console_lines":62}
111
+ {"time":"2026-04-30T15:58:18.336031498Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
112
+ {"time":"2026-04-30T15:58:33.00600962Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":101,"events_lines":2}
113
+ {"time":"2026-04-30T15:58:33.248302458Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
114
+ {"time":"2026-04-30T15:58:48.005259773Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":103,"events_lines":2}
115
+ {"time":"2026-04-30T15:58:48.223154078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
116
+ {"time":"2026-04-30T15:59:03.005663154Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":105,"events_lines":2}
117
+ {"time":"2026-04-30T15:59:03.331527437Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
118
+ {"time":"2026-04-30T15:59:18.005859686Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":107,"events_lines":2}
119
+ {"time":"2026-04-30T15:59:18.327832696Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
120
+ {"time":"2026-04-30T15:59:33.005751521Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":109,"events_lines":2}
121
+ {"time":"2026-04-30T15:59:33.231661802Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
122
+ {"time":"2026-04-30T15:59:48.0058735Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":111,"events_lines":2}
123
+ {"time":"2026-04-30T15:59:48.389276434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
124
+ {"time":"2026-04-30T16:00:03.005410784Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":113,"events_lines":2,"console_offset":104,"console_lines":1}
125
+ {"time":"2026-04-30T16:00:03.336154852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
126
+ {"time":"2026-04-30T16:00:18.005664291Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":115,"events_lines":2}
127
+ {"time":"2026-04-30T16:00:18.280680843Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
128
+ {"time":"2026-04-30T16:00:33.00615235Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":117,"events_lines":2,"console_offset":105,"console_lines":1}
129
+ {"time":"2026-04-30T16:00:33.455105614Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
130
+ {"time":"2026-04-30T16:00:48.005448932Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":119,"events_lines":2}
131
+ {"time":"2026-04-30T16:00:48.283748876Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
132
+ {"time":"2026-04-30T16:01:03.005754597Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":121,"events_lines":2,"console_offset":106,"console_lines":1}
133
+ {"time":"2026-04-30T16:01:03.315395049Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
134
+ {"time":"2026-04-30T16:01:18.00582113Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":123,"events_lines":2,"console_offset":107,"console_lines":1}
135
+ {"time":"2026-04-30T16:01:18.262007085Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
136
+ {"time":"2026-04-30T16:01:33.005333736Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":125,"events_lines":2}
137
+ {"time":"2026-04-30T16:01:36.411580847Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
138
+ {"time":"2026-04-30T16:01:48.005833683Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":127,"events_lines":2,"console_offset":108,"console_lines":1}
139
+ {"time":"2026-04-30T16:01:48.258582315Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
140
+ {"time":"2026-04-30T16:02:03.005388913Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":129,"events_lines":2}
141
+ {"time":"2026-04-30T16:02:03.372125765Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
142
+ {"time":"2026-04-30T16:02:18.005247118Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":131,"events_lines":2,"console_offset":109,"console_lines":1}
143
+ {"time":"2026-04-30T16:02:18.317183312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
144
+ {"time":"2026-04-30T16:02:33.005799613Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":133,"events_lines":2}
145
+ {"time":"2026-04-30T16:02:33.346512885Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
146
+ {"time":"2026-04-30T16:02:48.005972359Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":135,"events_lines":2,"console_offset":110,"console_lines":1}
147
+ {"time":"2026-04-30T16:02:48.292478674Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
148
+ {"time":"2026-04-30T16:03:03.005815451Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":137,"events_lines":2}
149
+ {"time":"2026-04-30T16:03:03.311579293Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
150
+ {"time":"2026-04-30T16:03:18.006076018Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":139,"events_lines":2,"console_offset":111,"console_lines":1}
151
+ {"time":"2026-04-30T16:03:18.36657545Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
152
+ {"time":"2026-04-30T16:03:33.006171398Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":141,"events_lines":2}
153
+ {"time":"2026-04-30T16:03:33.241840388Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
154
+ {"time":"2026-04-30T16:03:48.006249733Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":143,"events_lines":2,"console_offset":112,"console_lines":1}
155
+ {"time":"2026-04-30T16:03:48.259887407Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
156
+ {"time":"2026-04-30T16:04:03.005339695Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":145,"events_lines":2}
157
+ {"time":"2026-04-30T16:04:03.27582175Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
158
+ {"time":"2026-04-30T16:04:18.005581455Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":147,"events_lines":2,"console_offset":113,"console_lines":1}
159
+ {"time":"2026-04-30T16:04:18.288034464Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
160
+ {"time":"2026-04-30T16:04:33.005945307Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":149,"events_lines":2}
161
+ {"time":"2026-04-30T16:04:33.342042009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
162
+ {"time":"2026-04-30T16:04:48.005439586Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":151,"events_lines":2,"console_offset":114,"console_lines":1}
163
+ {"time":"2026-04-30T16:04:48.284113621Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
164
+ {"time":"2026-04-30T16:05:03.005918401Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":153,"events_lines":2}
165
+ {"time":"2026-04-30T16:05:03.362362203Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
166
+ {"time":"2026-04-30T16:05:18.006061791Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":155,"events_lines":2,"console_offset":115,"console_lines":1}
167
+ {"time":"2026-04-30T16:05:18.263228021Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
168
+ {"time":"2026-04-30T16:05:33.00606692Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":157,"events_lines":2,"console_offset":116,"console_lines":1}
169
+ {"time":"2026-04-30T16:05:33.269295919Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
170
+ {"time":"2026-04-30T16:05:48.005804469Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":159,"events_lines":2}
171
+ {"time":"2026-04-30T16:05:48.226351419Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
172
+ {"time":"2026-04-30T16:06:03.005490719Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":161,"events_lines":2,"console_offset":117,"console_lines":1}
173
+ {"time":"2026-04-30T16:06:03.373297826Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
174
+ {"time":"2026-04-30T16:06:18.0052578Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":163,"events_lines":2}
175
+ {"time":"2026-04-30T16:06:18.554840045Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
176
+ {"time":"2026-04-30T16:06:33.006350569Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":165,"events_lines":2,"console_offset":118,"console_lines":1}
177
+ {"time":"2026-04-30T16:06:33.306035387Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
178
+ {"time":"2026-04-30T16:06:48.005510661Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":167,"events_lines":2}
179
+ {"time":"2026-04-30T16:06:48.29614519Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
180
+ {"time":"2026-04-30T16:07:03.006057351Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":169,"events_lines":2,"console_offset":119,"console_lines":1}
181
+ {"time":"2026-04-30T16:07:03.290266518Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
182
+ {"time":"2026-04-30T16:07:18.005410321Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":171,"events_lines":2}
183
+ {"time":"2026-04-30T16:07:19.899874956Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
184
+ {"time":"2026-04-30T16:07:33.005623221Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":173,"events_lines":2,"console_offset":120,"console_lines":1}
185
+ {"time":"2026-04-30T16:07:33.254906167Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
186
+ {"time":"2026-04-30T16:07:48.00576193Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":175,"events_lines":2}
187
+ {"time":"2026-04-30T16:07:48.531708089Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
188
+ {"time":"2026-04-30T16:08:03.006138385Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":177,"events_lines":2,"console_offset":121,"console_lines":1}
189
+ {"time":"2026-04-30T16:08:03.483959164Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
190
+ {"time":"2026-04-30T16:08:18.005629853Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":179,"events_lines":2}
191
+ {"time":"2026-04-30T16:08:21.404167105Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
192
+ {"time":"2026-04-30T16:08:33.00566716Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":181,"events_lines":2,"console_offset":122,"console_lines":1}
193
+ {"time":"2026-04-30T16:08:37.020674895Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
194
+ {"time":"2026-04-30T16:08:48.006070975Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":183,"events_lines":2}
195
+ {"time":"2026-04-30T16:08:48.310024062Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
196
+ {"time":"2026-04-30T16:09:03.005330997Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":185,"events_lines":2,"console_offset":123,"console_lines":1}
197
+ {"time":"2026-04-30T16:09:03.318095113Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
198
+ {"time":"2026-04-30T16:09:18.005403065Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":187,"events_lines":2,"console_offset":124,"console_lines":1}
199
+ {"time":"2026-04-30T16:09:18.267587032Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
200
+ {"time":"2026-04-30T16:09:33.005343386Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":189,"events_lines":2}
201
+ {"time":"2026-04-30T16:09:33.264309108Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
202
+ {"time":"2026-04-30T16:09:48.005268735Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":191,"events_lines":2,"console_offset":125,"console_lines":1}
203
+ {"time":"2026-04-30T16:09:48.483572809Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
204
+ {"time":"2026-04-30T16:10:03.005624209Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":193,"events_lines":2}
205
+ {"time":"2026-04-30T16:10:03.266773875Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
206
+ {"time":"2026-04-30T16:10:18.005252482Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":195,"events_lines":2,"console_offset":126,"console_lines":1}
207
+ {"time":"2026-04-30T16:10:18.333882695Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
208
+ {"time":"2026-04-30T16:10:33.005487845Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":197,"events_lines":2}
209
+ {"time":"2026-04-30T16:10:33.48372572Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
210
+ {"time":"2026-04-30T16:10:48.006282798Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":199,"events_lines":2,"console_offset":127,"console_lines":1}
211
+ {"time":"2026-04-30T16:10:48.510713932Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
212
+ {"time":"2026-04-30T16:11:03.006165462Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":201,"events_lines":2}
213
+ {"time":"2026-04-30T16:11:03.273416556Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
214
+ {"time":"2026-04-30T16:11:18.006313505Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":203,"events_lines":2,"console_offset":128,"console_lines":1}
215
+ {"time":"2026-04-30T16:11:18.313582243Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
216
+ {"time":"2026-04-30T16:11:33.005809859Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":205,"events_lines":2}
217
+ {"time":"2026-04-30T16:11:33.266058134Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
218
+ {"time":"2026-04-30T16:11:48.005229129Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":207,"events_lines":2}
219
+ {"time":"2026-04-30T16:11:48.31233323Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
220
+ {"time":"2026-04-30T16:12:03.005390332Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":209,"events_lines":2}
221
+ {"time":"2026-04-30T16:12:03.30631602Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
222
+ {"time":"2026-04-30T16:12:18.005867938Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":211,"events_lines":2}
223
+ {"time":"2026-04-30T16:12:18.255049381Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
224
+ {"time":"2026-04-30T16:12:33.006185325Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":213,"events_lines":2}
225
+ {"time":"2026-04-30T16:12:33.294861318Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
226
+ {"time":"2026-04-30T16:12:48.005370844Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":215,"events_lines":2}
227
+ {"time":"2026-04-30T16:12:48.518465867Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
228
+ {"time":"2026-04-30T16:13:03.006106928Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":217,"events_lines":2}
229
+ {"time":"2026-04-30T16:13:03.374013574Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
230
+ {"time":"2026-04-30T16:13:18.006076934Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":219,"events_lines":2}
231
+ {"time":"2026-04-30T16:13:18.554997757Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
232
+ {"time":"2026-04-30T16:13:33.005580891Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":221,"events_lines":2}
233
+ {"time":"2026-04-30T16:13:33.277999882Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
234
+ {"time":"2026-04-30T16:13:48.005395894Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":223,"events_lines":2}
235
+ {"time":"2026-04-30T16:13:48.266523287Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
236
+ {"time":"2026-04-30T16:14:03.005230317Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":225,"events_lines":2}
237
+ {"time":"2026-04-30T16:14:03.259434189Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
238
+ {"time":"2026-04-30T16:14:18.005920672Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":227,"events_lines":2}
239
+ {"time":"2026-04-30T16:14:18.265956188Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
240
+ {"time":"2026-04-30T16:14:33.005453312Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":229,"events_lines":2}
241
+ {"time":"2026-04-30T16:14:33.254914114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
242
+ {"time":"2026-04-30T16:14:48.006176978Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":231,"events_lines":2}
243
+ {"time":"2026-04-30T16:14:48.596006932Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
244
+ {"time":"2026-04-30T16:15:03.005905051Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":233,"events_lines":2}
245
+ {"time":"2026-04-30T16:15:03.32015871Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
246
+ {"time":"2026-04-30T16:15:18.005463509Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":235,"events_lines":2}
247
+ {"time":"2026-04-30T16:15:18.300392447Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
248
+ {"time":"2026-04-30T16:15:33.006208076Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":237,"events_lines":2}
249
+ {"time":"2026-04-30T16:15:33.289061891Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
250
+ {"time":"2026-04-30T16:15:48.0058639Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":239,"events_lines":2}
251
+ {"time":"2026-04-30T16:15:48.225559338Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
252
+ {"time":"2026-04-30T16:16:03.005805103Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":241,"events_lines":2}
253
+ {"time":"2026-04-30T16:16:03.403059007Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
254
+ {"time":"2026-04-30T16:16:18.00607592Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":243,"events_lines":2}
255
+ {"time":"2026-04-30T16:16:18.298233833Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
256
+ {"time":"2026-04-30T16:16:33.005921479Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":245,"events_lines":2}
257
+ {"time":"2026-04-30T16:16:33.40759638Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
258
+ {"time":"2026-04-30T16:16:48.005699026Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":247,"events_lines":2}
259
+ {"time":"2026-04-30T16:16:48.382105845Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
260
+ {"time":"2026-04-30T16:17:03.00568782Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":249,"events_lines":2}
261
+ {"time":"2026-04-30T16:17:03.568478867Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
262
+ {"time":"2026-04-30T16:17:18.005902562Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":251,"events_lines":2}
263
+ {"time":"2026-04-30T16:17:19.765292202Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
264
+ {"time":"2026-04-30T16:17:33.00591422Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":253,"events_lines":2}
265
+ {"time":"2026-04-30T16:17:34.942957678Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
266
+ {"time":"2026-04-30T16:17:48.005610966Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":255,"events_lines":2}
267
+ {"time":"2026-04-30T16:17:48.274331563Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
268
+ {"time":"2026-04-30T16:18:03.005994672Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":257,"events_lines":2}
269
+ {"time":"2026-04-30T16:18:03.403865601Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
270
+ {"time":"2026-04-30T16:18:18.006057968Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":259,"events_lines":2}
271
+ {"time":"2026-04-30T16:18:18.279680333Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
272
+ {"time":"2026-04-30T16:18:33.0058726Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":261,"events_lines":2}
273
+ {"time":"2026-04-30T16:18:33.271891266Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
274
+ {"time":"2026-04-30T16:18:48.005247914Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":263,"events_lines":2}
275
+ {"time":"2026-04-30T16:18:48.282301263Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
276
+ {"time":"2026-04-30T16:19:03.005920173Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":265,"events_lines":2}
277
+ {"time":"2026-04-30T16:19:03.288129361Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
278
+ {"time":"2026-04-30T16:19:18.005490509Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":267,"events_lines":2}
279
+ {"time":"2026-04-30T16:19:18.505603429Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
280
+ {"time":"2026-04-30T16:19:33.005834746Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":269,"events_lines":2}
281
+ {"time":"2026-04-30T16:19:33.26390144Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
282
+ {"time":"2026-04-30T16:19:48.005960715Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":271,"events_lines":2}
283
+ {"time":"2026-04-30T16:19:48.236738057Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
284
+ {"time":"2026-04-30T16:20:03.005484362Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":273,"events_lines":2}
285
+ {"time":"2026-04-30T16:20:03.295830109Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
286
+ {"time":"2026-04-30T16:20:18.005857181Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":275,"events_lines":2,"console_offset":129,"console_lines":1}
287
+ {"time":"2026-04-30T16:20:18.540323564Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
288
+ {"time":"2026-04-30T16:20:33.006294778Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":277,"events_lines":2}
289
+ {"time":"2026-04-30T16:20:33.249663497Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
290
+ {"time":"2026-04-30T16:20:48.006265485Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":279,"events_lines":2}
291
+ {"time":"2026-04-30T16:20:48.747174156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
292
+ {"time":"2026-04-30T16:21:03.005392256Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":281,"events_lines":2}
293
+ {"time":"2026-04-30T16:21:03.42787335Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
294
+ {"time":"2026-04-30T16:21:18.005706909Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":283,"events_lines":2}
295
+ {"time":"2026-04-30T16:21:18.501301765Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
296
+ {"time":"2026-04-30T16:21:33.005995902Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":285,"events_lines":2}
297
+ {"time":"2026-04-30T16:21:33.278353331Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
298
+ {"time":"2026-04-30T16:21:48.005600781Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":287,"events_lines":2}
299
+ {"time":"2026-04-30T16:21:48.318034312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
300
+ {"time":"2026-04-30T16:22:03.006008703Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":289,"events_lines":2}
301
+ {"time":"2026-04-30T16:22:03.253541004Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
302
+ {"time":"2026-04-30T16:22:18.005686049Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":291,"events_lines":2}
303
+ {"time":"2026-04-30T16:22:18.293927471Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
304
+ {"time":"2026-04-30T16:22:33.005810015Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":293,"events_lines":2}
305
+ {"time":"2026-04-30T16:22:33.3447135Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
306
+ {"time":"2026-04-30T16:22:48.00616621Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":295,"events_lines":2}
307
+ {"time":"2026-04-30T16:22:48.683123074Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
308
+ {"time":"2026-04-30T16:23:03.005553522Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":297,"events_lines":2}
309
+ {"time":"2026-04-30T16:23:03.339204562Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
310
+ {"time":"2026-04-30T16:23:18.006085079Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":299,"events_lines":2}
311
+ {"time":"2026-04-30T16:23:18.524855766Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
312
+ {"time":"2026-04-30T16:23:33.005849401Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":301,"events_lines":2}
313
+ {"time":"2026-04-30T16:23:33.89177968Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
314
+ {"time":"2026-04-30T16:23:48.006131993Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":303,"events_lines":2}
315
+ {"time":"2026-04-30T16:23:48.274190498Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
316
+ {"time":"2026-04-30T16:24:03.005301851Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":305,"events_lines":2}
317
+ {"time":"2026-04-30T16:24:03.383717982Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
318
+ {"time":"2026-04-30T16:24:18.005998616Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":307,"events_lines":2}
319
+ {"time":"2026-04-30T16:24:18.303019353Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
320
+ {"time":"2026-04-30T16:24:33.00543874Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":309,"events_lines":2}
321
+ {"time":"2026-04-30T16:24:33.525188901Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
322
+ {"time":"2026-04-30T16:24:48.005941854Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":311,"events_lines":2}
323
+ {"time":"2026-04-30T16:24:48.394778356Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
324
+ {"time":"2026-04-30T16:25:03.005564763Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":313,"events_lines":2}
325
+ {"time":"2026-04-30T16:25:03.57377691Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
326
+ {"time":"2026-04-30T16:25:18.006088478Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":315,"events_lines":2}
327
+ {"time":"2026-04-30T16:25:24.955223684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
328
+ {"time":"2026-04-30T16:25:33.005718969Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":317,"events_lines":2}
329
+ {"time":"2026-04-30T16:25:33.507074839Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
330
+ {"time":"2026-04-30T16:25:48.005811665Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":319,"events_lines":2}
331
+ {"time":"2026-04-30T16:25:48.483564818Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
332
+ {"time":"2026-04-30T16:26:03.005900245Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":321,"events_lines":2}
333
+ {"time":"2026-04-30T16:26:03.479454119Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
334
+ {"time":"2026-04-30T16:26:18.005683345Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":323,"events_lines":2}
335
+ {"time":"2026-04-30T16:26:18.563506178Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
336
+ {"time":"2026-04-30T16:26:33.005883433Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":325,"events_lines":2}
337
+ {"time":"2026-04-30T16:26:33.301453417Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
338
+ {"time":"2026-04-30T16:26:48.006225424Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":327,"events_lines":2}
339
+ {"time":"2026-04-30T16:26:48.337510332Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
340
+ {"time":"2026-04-30T16:27:03.006192161Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":329,"events_lines":2}
341
+ {"time":"2026-04-30T16:27:03.366104277Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
342
+ {"time":"2026-04-30T16:27:18.005391743Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":331,"events_lines":2}
343
+ {"time":"2026-04-30T16:27:18.356553442Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
344
+ {"time":"2026-04-30T16:27:33.005424063Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":333,"events_lines":2}
345
+ {"time":"2026-04-30T16:27:33.253113591Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
346
+ {"time":"2026-04-30T16:27:48.005579037Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":335,"events_lines":2}
347
+ {"time":"2026-04-30T16:27:48.348411933Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
348
+ {"time":"2026-04-30T16:28:03.005692137Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":337,"events_lines":2}
349
+ {"time":"2026-04-30T16:28:03.563616393Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
350
+ {"time":"2026-04-30T16:28:18.005568442Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":339,"events_lines":2}
351
+ {"time":"2026-04-30T16:28:18.357222341Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
352
+ {"time":"2026-04-30T16:28:33.005275884Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":341,"events_lines":2}
353
+ {"time":"2026-04-30T16:28:33.260364183Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
354
+ {"time":"2026-04-30T16:28:48.006194461Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":343,"events_lines":2}
355
+ {"time":"2026-04-30T16:28:48.564777054Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
356
+ {"time":"2026-04-30T16:29:03.006086159Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":345,"events_lines":2}
357
+ {"time":"2026-04-30T16:29:03.437345224Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
358
+ {"time":"2026-04-30T16:29:18.005192781Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":347,"events_lines":2}
359
+ {"time":"2026-04-30T16:29:18.414149432Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
360
+ {"time":"2026-04-30T16:29:33.005992841Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":349,"events_lines":2}
361
+ {"time":"2026-04-30T16:29:33.249445096Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
362
+ {"time":"2026-04-30T16:29:48.005197459Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":351,"events_lines":2}
363
+ {"time":"2026-04-30T16:29:48.300125662Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
364
+ {"time":"2026-04-30T16:30:03.005837158Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":353,"events_lines":2}
365
+ {"time":"2026-04-30T16:30:03.644744616Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
366
+ {"time":"2026-04-30T16:30:18.005508626Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":355,"events_lines":2}
367
+ {"time":"2026-04-30T16:30:18.580291155Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
368
+ {"time":"2026-04-30T16:30:33.005399493Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":357,"events_lines":2}
369
+ {"time":"2026-04-30T16:30:33.771540244Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
370
+ {"time":"2026-04-30T16:30:48.005204342Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":359,"events_lines":2}
371
+ {"time":"2026-04-30T16:30:48.608178101Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
372
+ {"time":"2026-04-30T16:31:03.005973245Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":361,"events_lines":2}
373
+ {"time":"2026-04-30T16:31:03.308361921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
374
+ {"time":"2026-04-30T16:31:18.005892114Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":363,"events_lines":2}
375
+ {"time":"2026-04-30T16:31:18.355272966Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
376
+ {"time":"2026-04-30T16:31:33.005270522Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":365,"events_lines":2}
377
+ {"time":"2026-04-30T16:31:33.34313508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
378
+ {"time":"2026-04-30T16:31:48.006078324Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":367,"events_lines":2}
379
+ {"time":"2026-04-30T16:31:48.499480576Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
380
+ {"time":"2026-04-30T16:32:03.005836592Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":369,"events_lines":2}
381
+ {"time":"2026-04-30T16:32:03.576345985Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
382
+ {"time":"2026-04-30T16:32:18.005456653Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":371,"events_lines":2}
383
+ {"time":"2026-04-30T16:32:18.288499489Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
384
+ {"time":"2026-04-30T16:32:33.00595469Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":373,"events_lines":2}
385
+ {"time":"2026-04-30T16:32:34.924882358Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
386
+ {"time":"2026-04-30T16:32:48.006323454Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":375,"events_lines":2}
387
+ {"time":"2026-04-30T16:32:48.379617564Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
388
+ {"time":"2026-04-30T16:33:03.006170214Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":377,"events_lines":2}
389
+ {"time":"2026-04-30T16:33:03.447520967Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
390
+ {"time":"2026-04-30T16:33:18.005246971Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":379,"events_lines":2}
391
+ {"time":"2026-04-30T16:33:18.299434363Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
392
+ {"time":"2026-04-30T16:33:33.005214503Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":381,"events_lines":2}
393
+ {"time":"2026-04-30T16:33:33.257075494Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
394
+ {"time":"2026-04-30T16:33:48.006084968Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":383,"events_lines":2}
395
+ {"time":"2026-04-30T16:33:48.483998191Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
396
+ {"time":"2026-04-30T16:34:03.005713452Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":385,"events_lines":2}
397
+ {"time":"2026-04-30T16:34:03.500650625Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
398
+ {"time":"2026-04-30T16:34:18.005800623Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":387,"events_lines":2}
399
+ {"time":"2026-04-30T16:34:18.316031246Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
400
+ {"time":"2026-04-30T16:34:33.00549123Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":389,"events_lines":2}
401
+ {"time":"2026-04-30T16:34:33.918771966Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
402
+ {"time":"2026-04-30T16:34:48.006012051Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":391,"events_lines":2}
403
+ {"time":"2026-04-30T16:34:48.564277539Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
404
+ {"time":"2026-04-30T16:35:03.006151351Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":393,"events_lines":2}
405
+ {"time":"2026-04-30T16:35:03.368895686Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
406
+ {"time":"2026-04-30T16:35:18.005427716Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":395,"events_lines":2}
407
+ {"time":"2026-04-30T16:35:18.275481801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
408
+ {"time":"2026-04-30T16:35:33.00542287Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":397,"events_lines":2}
409
+ {"time":"2026-04-30T16:35:33.297633619Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
410
+ {"time":"2026-04-30T16:35:48.005882739Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":399,"events_lines":2}
411
+ {"time":"2026-04-30T16:35:48.278250933Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
412
+ {"time":"2026-04-30T16:36:03.005873897Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":401,"events_lines":2}
413
+ {"time":"2026-04-30T16:36:03.376615981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
414
+ {"time":"2026-04-30T16:36:18.00594233Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":403,"events_lines":2}
415
+ {"time":"2026-04-30T16:36:18.601102537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
416
+ {"time":"2026-04-30T16:36:33.005588671Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":405,"events_lines":2}
417
+ {"time":"2026-04-30T16:36:33.312505776Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
418
+ {"time":"2026-04-30T16:36:48.005250084Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":407,"events_lines":2}
419
+ {"time":"2026-04-30T16:36:48.250868777Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
420
+ {"time":"2026-04-30T16:37:03.006070514Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":409,"events_lines":2}
421
+ {"time":"2026-04-30T16:37:03.347460016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
422
+ {"time":"2026-04-30T16:37:18.005461213Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":411,"events_lines":2}
423
+ {"time":"2026-04-30T16:37:18.52436236Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
424
+ {"time":"2026-04-30T16:37:33.005399551Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":413,"events_lines":2}
425
+ {"time":"2026-04-30T16:37:33.273415881Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
426
+ {"time":"2026-04-30T16:37:48.005971169Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":415,"events_lines":2}
427
+ {"time":"2026-04-30T16:37:48.531172751Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
428
+ {"time":"2026-04-30T16:38:03.005637699Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":417,"events_lines":2}
429
+ {"time":"2026-04-30T16:38:03.377612391Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
430
+ {"time":"2026-04-30T16:38:18.005773724Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":419,"events_lines":2}
431
+ {"time":"2026-04-30T16:38:18.526478355Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
432
+ {"time":"2026-04-30T16:38:33.00575119Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":421,"events_lines":2}
433
+ {"time":"2026-04-30T16:38:33.499370558Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
434
+ {"time":"2026-04-30T16:38:48.005452373Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":423,"events_lines":2}
435
+ {"time":"2026-04-30T16:38:48.577282385Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
436
+ {"time":"2026-04-30T16:39:03.00634377Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":425,"events_lines":2}
437
+ {"time":"2026-04-30T16:39:03.506710523Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
438
+ {"time":"2026-04-30T16:39:18.005233857Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":427,"events_lines":2}
439
+ {"time":"2026-04-30T16:39:18.299173027Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
440
+ {"time":"2026-04-30T16:39:33.00521599Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":429,"events_lines":2}
441
+ {"time":"2026-04-30T16:39:33.26579483Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
442
+ {"time":"2026-04-30T16:39:48.005408329Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":431,"events_lines":2}
443
+ {"time":"2026-04-30T16:39:48.522643099Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
444
+ {"time":"2026-04-30T16:40:03.005792446Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":433,"events_lines":2}
445
+ {"time":"2026-04-30T16:40:03.377182201Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
446
+ {"time":"2026-04-30T16:40:18.006123152Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":435,"events_lines":2,"console_offset":130,"console_lines":65}
447
+ {"time":"2026-04-30T16:40:18.475683173Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
448
+ {"time":"2026-04-30T16:40:33.005469042Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":437,"events_lines":2}
449
+ {"time":"2026-04-30T16:40:33.279018667Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
450
+ {"time":"2026-04-30T16:40:48.006103434Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":439,"events_lines":2}
451
+ {"time":"2026-04-30T16:40:48.304756794Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
452
+ {"time":"2026-04-30T16:41:03.005857201Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":441,"events_lines":2}
453
+ {"time":"2026-04-30T16:41:03.325188483Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
454
+ {"time":"2026-04-30T16:41:18.005222352Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":443,"events_lines":2}
455
+ {"time":"2026-04-30T16:41:18.293214161Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
456
+ {"time":"2026-04-30T16:41:28.657147932Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
457
+ {"time":"2026-04-30T16:41:28.657330547Z","level":"INFO","msg":"filestream: sending request","total_files":3,"history_offset":24,"history_lines":1,"events_offset":445,"events_lines":1,"uploaded_len":3,"complete":true,"exit_code":1}
458
+ {"time":"2026-04-30T16:41:28.913871374Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
459
+ {"time":"2026-04-30T16:41:28.914009261Z","level":"INFO","msg":"stream: finishing up"}
460
+ {"time":"2026-04-30T16:41:28.914024484Z","level":"INFO","msg":"handler: closed"}
461
+ {"time":"2026-04-30T16:41:28.930157694Z","level":"INFO","msg":"sender: closed"}
462
+ {"time":"2026-04-30T16:41:28.930186758Z","level":"INFO","msg":"stream: all finished"}
epfl_final/wandb/run-20260430_154531-8xz0mg7d/logs/debug.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-04-30 15:45:31,913 INFO MainThread:242 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-04-30 15:45:31,913 INFO MainThread:242 [wandb_setup.py:_flush():81] Configure stats pid to 242
3
+ 2026-04-30 15:45:31,914 INFO MainThread:242 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-04-30 15:45:31,914 INFO MainThread:242 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_154531-8xz0mg7d/logs/debug.log
5
+ 2026-04-30 15:45:31,914 INFO MainThread:242 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_154531-8xz0mg7d/logs/debug-internal.log
6
+ 2026-04-30 15:45:31,914 INFO MainThread:242 [wandb_init.py:init():850] calling init triggers
7
+ 2026-04-30 15:45:31,914 INFO MainThread:242 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 3840, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_154528', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10005, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 14, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': True, 'sparse_layer_indices': None, '_wandb': {}}
9
+ 2026-04-30 15:45:31,914 INFO MainThread:242 [wandb_init.py:init():898] starting backend
10
+ 2026-04-30 15:45:32,133 INFO MainThread:242 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-04-30 15:45:32,443 INFO MainThread:242 [wandb_init.py:init():918] backend started and connected
12
+ 2026-04-30 15:45:32,444 INFO MainThread:242 [wandb_init.py:init():988] updated telemetry
13
+ 2026-04-30 15:45:32,445 INFO MainThread:242 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-04-30 15:45:32,921 INFO MainThread:242 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-04-30 15:45:33,000 INFO MainThread:242 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-04-30 15:45:33,000 INFO MainThread:242 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-04-30 15:45:33,000 INFO MainThread:242 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-04-30 15:45:33,000 INFO MainThread:242 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-04-30 15:45:33,003 INFO MainThread:242 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-04-30 16:41:26,148 INFO wandb-AsyncioManager-main:242 [service_client.py:_forward_responses():134] Reached EOF.
21
+ 2026-04-30 16:41:26,149 INFO wandb-AsyncioManager-main:242 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
epfl_final/wandb/run-20260430_154531-8xz0mg7d/run-8xz0mg7d.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:391892e53a1879b0347f959328cf9736fe0e53b734cfc6fe2c6cf3571b0bd820
3
+ size 473651
epfl_final/wandb/run-20260430_154759-yujhhjfq/files/output.log ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
2
+ Initializing model from scratch (random weights)...
3
+ Checkpoint activations: False
4
+
5
+
6
+ ================================================================================
7
+ MODEL ARCHITECTURE
8
+ ================================================================================
9
+ Model config:
10
+ vocab_size: 151936
11
+ hidden_size: 1024
12
+ num_hidden_layers: 28
13
+ num_attention_heads: 16
14
+ num_key_value_heads: 8
15
+ head_dim: 128
16
+ intermediate_size: 3072
17
+ max_position_embeddings: 32768
18
+ rope_theta: 1000000
19
+ attention_type: sparse_stochastic
20
+ sparse_block_size: 128
21
+ sparse_num_random_blocks: 9
22
+ sparse_mask_pool_size: 1000
23
+ sparse_max_seq_length: 3840
24
+
25
+ Parameter count:
26
+ Total parameters: 596,049,920
27
+ Trainable parameters: 596,049,920
28
+ Non-trainable parameters: 0
29
+ Model size (BF16): 1.19 GB
30
+
31
+ --------------------------------------------------------------------------------
32
+ LAYER-BY-LAYER SUMMARY
33
+ --------------------------------------------------------------------------------
34
+ /home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
35
+
36
+ SOLUTION: Use torch.compile(flex_attention)(...)
37
+
38
+ If you want to debug your score_mod/mask_mod, you can set:
39
+ torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
40
+
41
+ This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
42
+ _warn_once(
43
+ ================================================================================================================================================================
44
+ Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
45
+ ================================================================================================================================================================
46
+ Qwen3ForCausalLM [8, 3840] -- -- True
47
+ ├─Qwen3Model: 1-1 -- [8, 3840, 1024] -- True
48
+ │ └─Embedding: 2-1 [8, 3840] [8, 3840, 1024] 155,582,464 True
49
+ │ └─ModuleList: 2-2 -- -- -- True
50
+ │ │ └─Qwen3DecoderLayer: 3-1 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
51
+ │ │ └─Qwen3DecoderLayer: 3-2 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
52
+ │ │ └─Qwen3DecoderLayer: 3-3 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
53
+ │ │ └─Qwen3DecoderLayer: 3-4 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
54
+ │ │ └─Qwen3DecoderLayer: 3-5 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
55
+ │ │ └─Qwen3DecoderLayer: 3-6 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
56
+ │ │ └─Qwen3DecoderLayer: 3-7 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
57
+ │ │ └─Qwen3DecoderLayer: 3-8 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
58
+ │ │ └─Qwen3DecoderLayer: 3-9 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
59
+ │ │ └─Qwen3DecoderLayer: 3-10 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
60
+ │ │ └─Qwen3DecoderLayer: 3-11 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
61
+ │ │ └─Qwen3DecoderLayer: 3-12 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
62
+ │ │ └─Qwen3DecoderLayer: 3-13 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
63
+ │ │ └─Qwen3DecoderLayer: 3-14 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
64
+ │ │ └─Qwen3DecoderLayer: 3-15 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
65
+ │ │ └─Qwen3DecoderLayer: 3-16 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
66
+ │ │ └─Qwen3DecoderLayer: 3-17 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
67
+ │ │ └─Qwen3DecoderLayer: 3-18 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
68
+ │ │ └─Qwen3DecoderLayer: 3-19 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
69
+ │ │ └─Qwen3DecoderLayer: 3-20 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
70
+ │ │ └─Qwen3DecoderLayer: 3-21 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
71
+ │ │ └─Qwen3DecoderLayer: 3-22 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
72
+ │ │ └─Qwen3DecoderLayer: 3-23 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
73
+ │ │ └─Qwen3DecoderLayer: 3-24 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
74
+ │ │ └─Qwen3DecoderLayer: 3-25 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
75
+ │ │ └─Qwen3DecoderLayer: 3-26 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
76
+ │ │ └─Qwen3DecoderLayer: 3-27 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
77
+ │ │ └─Qwen3DecoderLayer: 3-28 [8, 3840, 1024] [8, 3840, 1024] 15,730,944 True
78
+ │ └─Qwen3RMSNorm: 2-3 [8, 3840, 1024] [8, 3840, 1024] 1,024 True
79
+ ├─Linear: 1-2 [8, 3840, 1024] [8, 3840, 151936] 155,582,464 True
80
+ ================================================================================================================================================================
81
+ Total params: 751,632,384
82
+ Trainable params: 751,632,384
83
+ Non-trainable params: 0
84
+ Total mult-adds (G): 6.01
85
+ ================================================================================================================================================================
86
+ Input size (MB): 0.25
87
+ Forward/backward pass size (MB): 78816.22
88
+ Params size (MB): 1503.26
89
+ Estimated Total Size (MB): 80319.73
90
+ ================================================================================================================================================================
91
+ ================================================================================
92
+
93
+ Checkpoint activations: False
94
+ Sparse validation compare enabled: running both sparse and dense validation passes
95
+ Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
96
+ Using pre-tokenized dataset (no runtime tokenization)
97
+ Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
98
+ Loaded 781148 pre-tokenized examples (stored_len=2048, pack_factor=2, packed_examples=390574)
99
+ Dataset split: 312459 train, 78115 validation
100
+ Training examples: 312459
101
+ Validation examples: 78115
102
+ Steps per epoch: 4882
103
+ Total training steps: 10005
104
+ Starting training...
105
+ Step 10/10005 | Loss: 11.8690 | LR: 5.00e-05 | 10478.3ms/step | 6.1 samples/s
106
+ Step 20/10005 | Loss: 10.6563 | LR: 1.00e-04 | 2975.0ms/step | 21.5 samples/s
107
+ Step 30/10005 | Loss: 9.6658 | LR: 1.50e-04 | 2972.3ms/step | 21.5 samples/s
108
+ Step 40/10005 | Loss: 8.5251 | LR: 2.00e-04 | 2971.2ms/step | 21.5 samples/s
109
+ Step 50/10005 | Loss: 8.0176 | LR: 2.50e-04 | 2972.4ms/step | 21.5 samples/s
110
+ Step 60/10005 | Loss: 7.8423 | LR: 3.00e-04 | 2978.4ms/step | 21.5 samples/s
111
+ Step 70/10005 | Loss: 7.6645 | LR: 3.50e-04 | 2979.2ms/step | 21.5 samples/s
112
+ Step 80/10005 | Loss: 7.4411 | LR: 4.00e-04 | 2985.2ms/step | 21.4 samples/s
113
+ Step 90/10005 | Loss: 7.2237 | LR: 4.50e-04 | 2991.5ms/step | 21.4 samples/s
114
+ Step 100/10005 | Loss: 7.0741 | LR: 5.00e-04 | 2993.6ms/step | 21.4 samples/s
115
+ Step 110/10005 | Loss: 6.9384 | LR: 5.00e-04 | 3159.4ms/step | 20.3 samples/s
116
+ Step 120/10005 | Loss: 6.8175 | LR: 5.00e-04 | 2992.2ms/step | 21.4 samples/s
117
+ Step 130/10005 | Loss: 6.7281 | LR: 5.00e-04 | 2994.8ms/step | 21.4 samples/s
118
+ Step 140/10005 | Loss: 6.6708 | LR: 5.00e-04 | 2991.6ms/step | 21.4 samples/s
119
+ Step 150/10005 | Loss: 6.5577 | LR: 5.00e-04 | 2999.9ms/step | 21.3 samples/s
120
+ Step 160/10005 | Loss: 6.4702 | LR: 5.00e-04 | 2996.6ms/step | 21.4 samples/s
121
+ Step 170/10005 | Loss: 6.4012 | LR: 5.00e-04 | 2993.5ms/step | 21.4 samples/s
122
+ Step 180/10005 | Loss: 6.4000 | LR: 5.00e-04 | 2998.2ms/step | 21.3 samples/s
123
+ Step 190/10005 | Loss: 6.2743 | LR: 5.00e-04 | 2993.0ms/step | 21.4 samples/s
124
+ Step 200/10005 | Loss: 6.1858 | LR: 5.00e-04 | 2993.8ms/step | 21.4 samples/s
125
+ Step 210/10005 | Loss: 6.1445 | LR: 5.00e-04 | 2996.8ms/step | 21.4 samples/s
126
+ Step 220/10005 | Loss: 6.0908 | LR: 5.00e-04 | 2990.8ms/step | 21.4 samples/s
127
+ Step 230/10005 | Loss: 6.0202 | LR: 5.00e-04 | 2997.0ms/step | 21.4 samples/s
128
+ Step 240/10005 | Loss: 5.9320 | LR: 5.00e-04 | 2996.2ms/step | 21.4 samples/s
129
+ Step 250/10005 | Loss: 5.9377 | LR: 5.00e-04 | 2992.6ms/step | 21.4 samples/s
130
+ Validation at step 250: Loss = 5.9856, Perplexity = 397.65
131
+ Dense validation compare at step 250: Loss = 6.0030, Perplexity = 404.63
132
+ Traceback (most recent call last):
133
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1069, in <module>
134
+ main()
135
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1065, in main
136
+ train(config)
137
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 926, in train
138
+ loss.backward()
139
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_tensor.py", line 631, in backward
140
+ torch.autograd.backward(
141
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/__init__.py", line 381, in backward
142
+ _engine_run_backward(
143
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
144
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
145
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 317, in apply
146
+ return user_fn(self, *args)
147
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2748, in backward
148
+ return impl_fn()
149
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2734, in impl_fn
150
+ out = CompiledFunction._backward_impl(ctx, all_args)
151
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2918, in _backward_impl
152
+ out = call_func_at_runtime_with_args(
153
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 138, in call_func_at_runtime_with_args
154
+ out = normalize_as_list(f(args))
155
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
156
+ return fn(*args, **kwargs)
157
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 656, in __call__
158
+ return self.current_callable(inputs)
159
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3401, in run
160
+ out = model(new_inputs)
161
+ File "/tmp/torchinductor_navasard/gb/cgbnsvxvqkogviif253ijeog4w2n2lahagxmijbdxhjzze3wsfej.py", line 305, in call
162
+ buf1 = empty_strided_cuda((8, 3840, 151936), (583434240, 151936, 1), torch.bfloat16)
163
+ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 8.70 GiB. GPU 0 has a total capacity of 79.18 GiB of which 2.90 GiB is free. Including non-PyTorch memory, this process has 76.23 GiB memory in use. Of the allocated memory 62.18 GiB is allocated by PyTorch, and 13.31 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
164
+ Traceback (most recent call last):
165
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1069, in <module>
166
+ main()
167
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1065, in main
168
+ train(config)
169
+ File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 926, in train
170
+ loss.backward()
171
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_tensor.py", line 631, in backward
172
+ torch.autograd.backward(
173
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/__init__.py", line 381, in backward
174
+ _engine_run_backward(
175
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/graph.py", line 869, in _engine_run_backward
176
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
177
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/autograd/function.py", line 317, in apply
178
+ return user_fn(self, *args)
179
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2748, in backward
180
+ return impl_fn()
181
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2734, in impl_fn
182
+ out = CompiledFunction._backward_impl(ctx, all_args)
183
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2918, in _backward_impl
184
+ out = call_func_at_runtime_with_args(
185
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_functorch/_aot_autograd/utils.py", line 138, in call_func_at_runtime_with_args
186
+ out = normalize_as_list(f(args))
187
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 1263, in _fn
188
+ return fn(*args, **kwargs)
189
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/output_code.py", line 656, in __call__
190
+ return self.current_callable(inputs)
191
+ File "/home/navasard/.local/lib/python3.10/site-packages/torch/_inductor/utils.py", line 3401, in run
192
+ out = model(new_inputs)
193
+ File "/tmp/torchinductor_navasard/gb/cgbnsvxvqkogviif253ijeog4w2n2lahagxmijbdxhjzze3wsfej.py", line 305, in call
194
+ buf1 = empty_strided_cuda((8, 3840, 151936), (583434240, 151936, 1), torch.bfloat16)
195
+ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 8.70 GiB. GPU 0 has a total capacity of 79.18 GiB of which 2.90 GiB is free. Including non-PyTorch memory, this process has 76.23 GiB memory in use. Of the allocated memory 62.18 GiB is allocated by PyTorch, and 13.31 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf)
epfl_final/wandb/run-20260430_154759-yujhhjfq/files/requirements.txt ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pytz==2026.1.post1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ tzdata==2026.2
9
+ typing_extensions==4.15.0
10
+ triton==3.6.0
11
+ tqdm==4.67.3
12
+ torchinfo==1.8.0
13
+ sympy==1.14.0
14
+ smmap==5.0.3
15
+ six==1.17.0
16
+ shellingham==1.5.4
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.2.6
38
+ networkx==3.4.2
39
+ mdurl==0.1.2
40
+ MarkupSafe==3.0.3
41
+ idna==3.13
42
+ hf-xet==1.4.3
43
+ h11==0.16.0
44
+ fsspec==2026.2.0
45
+ frozenlist==1.8.0
46
+ filelock==3.29.0
47
+ dill==0.4.1
48
+ cuda-pathfinder==1.5.4
49
+ click==8.3.3
50
+ charset-normalizer==3.4.7
51
+ certifi==2026.4.22
52
+ attrs==26.1.0
53
+ async-timeout==5.0.1
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ typing-inspection==0.4.2
58
+ sentry-sdk==2.58.0
59
+ requests==2.33.1
60
+ python-dateutil==2.9.0.post0
61
+ pydantic_core==2.46.3
62
+ nvidia-cusparse==12.6.3.3
63
+ nvidia-cufft==12.0.0.61
64
+ nvidia-cudnn-cu13==9.19.0.56
65
+ multiprocess==0.70.19
66
+ multidict==6.7.1
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ exceptiongroup==1.3.1
72
+ cuda-bindings==13.2.0
73
+ aiosignal==1.4.0
74
+ yarl==1.23.0
75
+ rich==15.0.0
76
+ pydantic==2.13.3
77
+ pandas==2.3.3
78
+ nvidia-cusolver==12.0.4.66
79
+ GitPython==3.1.49
80
+ anyio==4.13.0
81
+ wandb==0.26.1
82
+ typer==0.25.0
83
+ httpx==0.28.1
84
+ aiohttp==3.13.5
85
+ torch==2.11.0
86
+ huggingface_hub==1.13.0
87
+ tokenizers==0.22.2
88
+ datasets==4.8.5
89
+ accelerate==1.13.0
90
+ transformers==5.7.0
91
+ PyGObject==3.42.1
92
+ dbus-python==1.2.18
93
+ distro==1.7.0
94
+ pip==22.0.2
95
+ setuptools==59.6.0
96
+ ssh-import-id==5.11
97
+ wheel==0.37.1
epfl_final/wandb/run-20260430_154759-yujhhjfq/files/wandb-metadata.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.10.12",
4
+ "startedAt": "2026-04-30T15:47:59.246695Z",
5
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
6
+ "codePath": "SparseAtt/train.py",
7
+ "codePathLocal": "SparseAtt/train.py",
8
+ "email": "ruben.navasardyan@mail.utoronto.ca",
9
+ "root": "/mloscratch/homes/navasard/sparse",
10
+ "host": "h100-workspace-7-0-1",
11
+ "executable": "/usr/bin/python3",
12
+ "cpu_count": 96,
13
+ "cpu_count_logical": 192,
14
+ "gpu": "NVIDIA H100 80GB HBM3",
15
+ "gpu_count": 1,
16
+ "disk": {
17
+ "/": {
18
+ "total": "9374931091456",
19
+ "used": "6049983488000"
20
+ }
21
+ },
22
+ "memory": {
23
+ "total": "1623005847552"
24
+ },
25
+ "gpu_nvidia": [
26
+ {
27
+ "name": "NVIDIA H100 80GB HBM3",
28
+ "memoryTotal": "85520809984",
29
+ "cudaCores": 16896,
30
+ "architecture": "Hopper",
31
+ "uuid": "GPU-410cba62-f098-3454-c960-ebc55bd081af"
32
+ }
33
+ ],
34
+ "cudaVersion": "13.0",
35
+ "writerId": "wlkifov2nkv7u22tti0lvhg8pz6y8y20"
36
+ }
epfl_final/wandb/run-20260430_154759-yujhhjfq/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"eval_dense_compare/perplexity":404.62966578311017,"speed/steps_per_sec":0.334154871963508,"speed/ms_per_step":2992.62433052063,"speed/samples_per_sec":21.385911805664513,"_wandb":{"runtime":3149},"_step":250,"_runtime":3149.028401564,"train/global_step":250,"train/learning_rate":0.0004997171205507588,"_timestamp":1.7775671484049997e+09,"train/epoch":0,"eval_dense_compare/loss":6.002972243309021,"eval/loss":5.985563106179237,"eval/perplexity":397.64637534773306,"train/loss":5.9376681566238405}
epfl_final/wandb/run-20260430_154759-yujhhjfq/logs/debug-internal.log ADDED
@@ -0,0 +1,434 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-04-30T15:47:59.509430809Z","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-04-30T15:47:59.509853643Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
3
+ {"time":"2026-04-30T15:47:59.793521227Z","level":"INFO","msg":"stream: created new stream","id":"yujhhjfq"}
4
+ {"time":"2026-04-30T15:47:59.793630391Z","level":"INFO","msg":"handler: started"}
5
+ {"time":"2026-04-30T15:47:59.796127394Z","level":"INFO","msg":"stream: started"}
6
+ {"time":"2026-04-30T15:47:59.796168806Z","level":"INFO","msg":"writer: started","stream_id":"yujhhjfq"}
7
+ {"time":"2026-04-30T15:47:59.796202597Z","level":"INFO","msg":"sender: started"}
8
+ {"time":"2026-04-30T15:48:00.244140591Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
9
+ {"time":"2026-04-30T15:48:00.708885008Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-04-30T15:48:15.244753197Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":2,"uploaded_len":2}
11
+ {"time":"2026-04-30T15:48:15.552252966Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-04-30T15:48:30.244605664Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2}
13
+ {"time":"2026-04-30T15:48:30.533027244Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
14
+ {"time":"2026-04-30T15:48:45.24453053Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":3,"events_lines":2}
15
+ {"time":"2026-04-30T15:48:45.535007107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
16
+ {"time":"2026-04-30T15:49:00.244859801Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":5,"events_lines":2}
17
+ {"time":"2026-04-30T15:49:00.568195924Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
18
+ {"time":"2026-04-30T15:49:15.244478386Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":7,"events_lines":2}
19
+ {"time":"2026-04-30T15:49:15.629142491Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
20
+ {"time":"2026-04-30T15:49:30.244418784Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":9,"events_lines":2}
21
+ {"time":"2026-04-30T15:49:30.539470669Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
22
+ {"time":"2026-04-30T15:49:45.244993092Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":11,"events_lines":2}
23
+ {"time":"2026-04-30T15:49:45.505885231Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
24
+ {"time":"2026-04-30T15:50:00.249851461Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":13,"events_lines":2}
25
+ {"time":"2026-04-30T15:50:00.575725649Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
26
+ {"time":"2026-04-30T15:50:15.24419581Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":15,"events_lines":2}
27
+ {"time":"2026-04-30T15:50:17.155421973Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
28
+ {"time":"2026-04-30T15:50:30.245155927Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":17,"events_lines":2}
29
+ {"time":"2026-04-30T15:50:30.495604221Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
30
+ {"time":"2026-04-30T15:50:45.245074522Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":19,"events_lines":2}
31
+ {"time":"2026-04-30T15:50:45.588635202Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
32
+ {"time":"2026-04-30T15:51:00.244917804Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":21,"events_lines":2}
33
+ {"time":"2026-04-30T15:51:00.723343708Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
34
+ {"time":"2026-04-30T15:51:15.244312263Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":23,"events_lines":2}
35
+ {"time":"2026-04-30T15:51:15.485286626Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
36
+ {"time":"2026-04-30T15:51:30.244811786Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":25,"events_lines":2}
37
+ {"time":"2026-04-30T15:51:30.585256936Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
38
+ {"time":"2026-04-30T15:51:45.24460985Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":27,"events_lines":2}
39
+ {"time":"2026-04-30T15:51:47.023110336Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
40
+ {"time":"2026-04-30T15:52:00.244708215Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":29,"events_lines":2}
41
+ {"time":"2026-04-30T15:52:00.730661272Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
42
+ {"time":"2026-04-30T15:52:15.244576324Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":31,"events_lines":2}
43
+ {"time":"2026-04-30T15:52:15.513469818Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
44
+ {"time":"2026-04-30T15:52:30.244807778Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":33,"events_lines":2}
45
+ {"time":"2026-04-30T15:52:30.530829226Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
46
+ {"time":"2026-04-30T15:52:45.244965511Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":35,"events_lines":2}
47
+ {"time":"2026-04-30T15:52:45.566845111Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
48
+ {"time":"2026-04-30T15:53:00.244539821Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":37,"events_lines":2}
49
+ {"time":"2026-04-30T15:53:00.608669223Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
50
+ {"time":"2026-04-30T15:53:15.244538245Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":39,"events_lines":2}
51
+ {"time":"2026-04-30T15:53:15.536340319Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
52
+ {"time":"2026-04-30T15:53:30.244940615Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":41,"events_lines":2}
53
+ {"time":"2026-04-30T15:53:30.54457558Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
54
+ {"time":"2026-04-30T15:53:45.244891839Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":43,"events_lines":2}
55
+ {"time":"2026-04-30T15:53:45.597513922Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
56
+ {"time":"2026-04-30T15:54:00.244551075Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":45,"events_lines":2}
57
+ {"time":"2026-04-30T15:54:00.614275345Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
58
+ {"time":"2026-04-30T15:54:15.244154027Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":47,"events_lines":2}
59
+ {"time":"2026-04-30T15:54:15.550362076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
60
+ {"time":"2026-04-30T15:54:30.244962166Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":49,"events_lines":2}
61
+ {"time":"2026-04-30T15:54:30.542830432Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
62
+ {"time":"2026-04-30T15:54:45.244710535Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":51,"events_lines":2}
63
+ {"time":"2026-04-30T15:54:47.119032148Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
64
+ {"time":"2026-04-30T15:55:00.244482611Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":53,"events_lines":2}
65
+ {"time":"2026-04-30T15:55:00.570436067Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
66
+ {"time":"2026-04-30T15:55:15.244806603Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":55,"events_lines":2}
67
+ {"time":"2026-04-30T15:55:15.523022812Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
68
+ {"time":"2026-04-30T15:55:30.24455924Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":57,"events_lines":2}
69
+ {"time":"2026-04-30T15:55:30.614235838Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
70
+ {"time":"2026-04-30T15:55:45.244286818Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":59,"events_lines":2}
71
+ {"time":"2026-04-30T15:55:45.526353859Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
72
+ {"time":"2026-04-30T15:56:00.244935648Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":61,"events_lines":2,"console_offset":3,"console_lines":101}
73
+ {"time":"2026-04-30T15:56:00.592182263Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
74
+ {"time":"2026-04-30T15:56:15.244865259Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":63,"events_lines":2}
75
+ {"time":"2026-04-30T15:56:15.597559671Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
76
+ {"time":"2026-04-30T15:56:30.244141842Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":65,"events_lines":2}
77
+ {"time":"2026-04-30T15:56:30.545478929Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
78
+ {"time":"2026-04-30T15:56:45.244901538Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":67,"events_lines":2}
79
+ {"time":"2026-04-30T15:56:45.506057583Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
80
+ {"time":"2026-04-30T15:57:00.244499993Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":69,"events_lines":2}
81
+ {"time":"2026-04-30T15:57:00.534033576Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
82
+ {"time":"2026-04-30T15:57:15.244638507Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":71,"events_lines":2}
83
+ {"time":"2026-04-30T15:57:15.617347712Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
84
+ {"time":"2026-04-30T15:57:30.244939015Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":73,"events_lines":2}
85
+ {"time":"2026-04-30T15:57:30.493296195Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
86
+ {"time":"2026-04-30T15:57:45.244729898Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":75,"events_lines":2,"console_offset":104,"console_lines":1}
87
+ {"time":"2026-04-30T15:57:45.571830144Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
88
+ {"time":"2026-04-30T15:58:00.24485946Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":77,"events_lines":2}
89
+ {"time":"2026-04-30T15:58:00.561359145Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
90
+ {"time":"2026-04-30T15:58:15.244491545Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":79,"events_lines":2,"console_offset":105,"console_lines":1}
91
+ {"time":"2026-04-30T15:58:15.556751355Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
92
+ {"time":"2026-04-30T15:58:30.244981186Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":81,"events_lines":2}
93
+ {"time":"2026-04-30T15:58:30.609368175Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
94
+ {"time":"2026-04-30T15:58:45.244877778Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":83,"events_lines":2,"console_offset":106,"console_lines":1}
95
+ {"time":"2026-04-30T15:58:45.575748035Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
96
+ {"time":"2026-04-30T15:59:00.244546719Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":85,"events_lines":2}
97
+ {"time":"2026-04-30T15:59:00.579986406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
98
+ {"time":"2026-04-30T15:59:15.244994307Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":87,"events_lines":2,"console_offset":107,"console_lines":1}
99
+ {"time":"2026-04-30T15:59:15.622708744Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
100
+ {"time":"2026-04-30T15:59:30.245108385Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":89,"events_lines":2}
101
+ {"time":"2026-04-30T15:59:30.531105377Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
102
+ {"time":"2026-04-30T15:59:45.244518658Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":91,"events_lines":2,"console_offset":108,"console_lines":1}
103
+ {"time":"2026-04-30T15:59:45.584814664Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
104
+ {"time":"2026-04-30T16:00:00.244700638Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":93,"events_lines":2}
105
+ {"time":"2026-04-30T16:00:00.848336312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
106
+ {"time":"2026-04-30T16:00:15.244727144Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":95,"events_lines":2,"console_offset":109,"console_lines":1}
107
+ {"time":"2026-04-30T16:00:15.553274439Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
108
+ {"time":"2026-04-30T16:00:30.244408364Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":97,"events_lines":2}
109
+ {"time":"2026-04-30T16:00:30.502032975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
110
+ {"time":"2026-04-30T16:00:45.244662962Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":99,"events_lines":2,"console_offset":110,"console_lines":1}
111
+ {"time":"2026-04-30T16:00:45.673741004Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
112
+ {"time":"2026-04-30T16:01:00.244845272Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":101,"events_lines":2}
113
+ {"time":"2026-04-30T16:01:00.604544264Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
114
+ {"time":"2026-04-30T16:01:15.244762692Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":103,"events_lines":2,"console_offset":111,"console_lines":1}
115
+ {"time":"2026-04-30T16:01:15.57615372Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
116
+ {"time":"2026-04-30T16:01:30.244779564Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":105,"events_lines":2}
117
+ {"time":"2026-04-30T16:01:30.511368403Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
118
+ {"time":"2026-04-30T16:01:45.244923617Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":107,"events_lines":2,"console_offset":112,"console_lines":1}
119
+ {"time":"2026-04-30T16:01:45.477410687Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
120
+ {"time":"2026-04-30T16:02:00.244592288Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":109,"events_lines":2}
121
+ {"time":"2026-04-30T16:02:00.624922475Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
122
+ {"time":"2026-04-30T16:02:15.24498947Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":111,"events_lines":2,"console_offset":113,"console_lines":1}
123
+ {"time":"2026-04-30T16:02:15.575072988Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
124
+ {"time":"2026-04-30T16:02:30.244422627Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":113,"events_lines":2}
125
+ {"time":"2026-04-30T16:02:30.563203492Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
126
+ {"time":"2026-04-30T16:02:45.244729915Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":115,"events_lines":2,"console_offset":114,"console_lines":1}
127
+ {"time":"2026-04-30T16:02:45.554016488Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
128
+ {"time":"2026-04-30T16:03:00.244874268Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":117,"events_lines":2}
129
+ {"time":"2026-04-30T16:03:00.553758568Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
130
+ {"time":"2026-04-30T16:03:15.244446013Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":119,"events_lines":2,"console_offset":115,"console_lines":1}
131
+ {"time":"2026-04-30T16:03:15.508473943Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
132
+ {"time":"2026-04-30T16:03:30.244829085Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":121,"events_lines":2}
133
+ {"time":"2026-04-30T16:03:30.76558403Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
134
+ {"time":"2026-04-30T16:03:45.245025145Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":123,"events_lines":2,"console_offset":116,"console_lines":1}
135
+ {"time":"2026-04-30T16:03:45.5315579Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
136
+ {"time":"2026-04-30T16:04:00.244593896Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":125,"events_lines":2}
137
+ {"time":"2026-04-30T16:04:00.546128699Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
138
+ {"time":"2026-04-30T16:04:15.245000482Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":127,"events_lines":2,"console_offset":117,"console_lines":1}
139
+ {"time":"2026-04-30T16:04:15.577301084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
140
+ {"time":"2026-04-30T16:04:30.244475563Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":129,"events_lines":2}
141
+ {"time":"2026-04-30T16:04:30.536558257Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
142
+ {"time":"2026-04-30T16:04:45.244776501Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":131,"events_lines":2,"console_offset":118,"console_lines":1}
143
+ {"time":"2026-04-30T16:04:45.56904403Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
144
+ {"time":"2026-04-30T16:05:00.24443122Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":133,"events_lines":2}
145
+ {"time":"2026-04-30T16:05:00.641837475Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
146
+ {"time":"2026-04-30T16:05:15.244538751Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":135,"events_lines":2,"console_offset":119,"console_lines":1}
147
+ {"time":"2026-04-30T16:05:15.570931328Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
148
+ {"time":"2026-04-30T16:05:30.244775067Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":137,"events_lines":2}
149
+ {"time":"2026-04-30T16:05:30.568550332Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
150
+ {"time":"2026-04-30T16:05:45.244752796Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":139,"events_lines":2,"console_offset":120,"console_lines":1}
151
+ {"time":"2026-04-30T16:05:45.551760987Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
152
+ {"time":"2026-04-30T16:06:00.244557504Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":141,"events_lines":2}
153
+ {"time":"2026-04-30T16:06:00.594972917Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
154
+ {"time":"2026-04-30T16:06:15.245122351Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":143,"events_lines":2,"console_offset":121,"console_lines":1}
155
+ {"time":"2026-04-30T16:06:15.601340925Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
156
+ {"time":"2026-04-30T16:06:30.244179739Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":145,"events_lines":2}
157
+ {"time":"2026-04-30T16:06:30.535491013Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
158
+ {"time":"2026-04-30T16:06:45.245090365Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":147,"events_lines":2,"console_offset":122,"console_lines":1}
159
+ {"time":"2026-04-30T16:06:45.547291495Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
160
+ {"time":"2026-04-30T16:07:00.244760491Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":149,"events_lines":2}
161
+ {"time":"2026-04-30T16:07:00.568056607Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
162
+ {"time":"2026-04-30T16:07:15.244569385Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":151,"events_lines":2,"console_offset":123,"console_lines":1}
163
+ {"time":"2026-04-30T16:07:15.804738501Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
164
+ {"time":"2026-04-30T16:07:30.245103516Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":153,"events_lines":2}
165
+ {"time":"2026-04-30T16:07:30.521514262Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
166
+ {"time":"2026-04-30T16:07:45.244506742Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":155,"events_lines":2,"console_offset":124,"console_lines":1}
167
+ {"time":"2026-04-30T16:07:45.472466069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
168
+ {"time":"2026-04-30T16:08:00.244705671Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":157,"events_lines":2}
169
+ {"time":"2026-04-30T16:08:00.625181261Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
170
+ {"time":"2026-04-30T16:08:15.244358281Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":159,"events_lines":2,"console_offset":125,"console_lines":1}
171
+ {"time":"2026-04-30T16:08:15.586801843Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
172
+ {"time":"2026-04-30T16:08:30.244949338Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":161,"events_lines":2}
173
+ {"time":"2026-04-30T16:08:30.591240508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
174
+ {"time":"2026-04-30T16:08:45.244445553Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":163,"events_lines":2,"console_offset":126,"console_lines":1}
175
+ {"time":"2026-04-30T16:08:45.484899542Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
176
+ {"time":"2026-04-30T16:09:00.244216099Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":165,"events_lines":2}
177
+ {"time":"2026-04-30T16:09:00.632976189Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
178
+ {"time":"2026-04-30T16:09:15.244824823Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":167,"events_lines":2,"console_offset":127,"console_lines":1}
179
+ {"time":"2026-04-30T16:09:15.799752201Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
180
+ {"time":"2026-04-30T16:09:30.245085324Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":169,"events_lines":2}
181
+ {"time":"2026-04-30T16:09:32.02838574Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
182
+ {"time":"2026-04-30T16:09:45.244760508Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":171,"events_lines":2,"console_offset":128,"console_lines":1}
183
+ {"time":"2026-04-30T16:09:45.517163284Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
184
+ {"time":"2026-04-30T16:10:00.244997393Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":173,"events_lines":2}
185
+ {"time":"2026-04-30T16:10:00.528760671Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
186
+ {"time":"2026-04-30T16:10:15.244364555Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":175,"events_lines":2}
187
+ {"time":"2026-04-30T16:10:15.543838076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
188
+ {"time":"2026-04-30T16:10:30.244884565Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":177,"events_lines":2}
189
+ {"time":"2026-04-30T16:10:30.82114795Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
190
+ {"time":"2026-04-30T16:10:45.245150214Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":179,"events_lines":2}
191
+ {"time":"2026-04-30T16:10:45.520833979Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
192
+ {"time":"2026-04-30T16:11:00.244352659Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":181,"events_lines":2}
193
+ {"time":"2026-04-30T16:11:01.062327187Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
194
+ {"time":"2026-04-30T16:11:15.24449296Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":183,"events_lines":2}
195
+ {"time":"2026-04-30T16:11:15.860308942Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
196
+ {"time":"2026-04-30T16:11:30.244928131Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":185,"events_lines":2}
197
+ {"time":"2026-04-30T16:11:30.529802005Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
198
+ {"time":"2026-04-30T16:11:45.245097196Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":187,"events_lines":2}
199
+ {"time":"2026-04-30T16:11:45.830116347Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
200
+ {"time":"2026-04-30T16:12:00.244794062Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":189,"events_lines":2}
201
+ {"time":"2026-04-30T16:12:00.612260578Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
202
+ {"time":"2026-04-30T16:12:15.244479921Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":191,"events_lines":2}
203
+ {"time":"2026-04-30T16:12:15.637615541Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
204
+ {"time":"2026-04-30T16:12:30.244128155Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":193,"events_lines":2}
205
+ {"time":"2026-04-30T16:12:30.572612419Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
206
+ {"time":"2026-04-30T16:12:45.244423258Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":195,"events_lines":2}
207
+ {"time":"2026-04-30T16:12:45.758687894Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
208
+ {"time":"2026-04-30T16:13:00.244385742Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":197,"events_lines":2}
209
+ {"time":"2026-04-30T16:13:00.657893614Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
210
+ {"time":"2026-04-30T16:13:15.244229228Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":199,"events_lines":2}
211
+ {"time":"2026-04-30T16:13:15.56252672Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
212
+ {"time":"2026-04-30T16:13:30.245188386Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":201,"events_lines":2}
213
+ {"time":"2026-04-30T16:13:30.601976294Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
214
+ {"time":"2026-04-30T16:13:45.244437962Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":203,"events_lines":2}
215
+ {"time":"2026-04-30T16:13:46.186540594Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
216
+ {"time":"2026-04-30T16:14:00.244883621Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":205,"events_lines":2}
217
+ {"time":"2026-04-30T16:14:00.806000338Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
218
+ {"time":"2026-04-30T16:14:15.244277493Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":207,"events_lines":2}
219
+ {"time":"2026-04-30T16:14:15.539709265Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
220
+ {"time":"2026-04-30T16:14:30.244884954Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":209,"events_lines":2}
221
+ {"time":"2026-04-30T16:14:30.620968369Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
222
+ {"time":"2026-04-30T16:14:45.244643142Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":211,"events_lines":2}
223
+ {"time":"2026-04-30T16:14:45.537303409Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
224
+ {"time":"2026-04-30T16:15:00.244351155Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":213,"events_lines":2}
225
+ {"time":"2026-04-30T16:15:00.582432357Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
226
+ {"time":"2026-04-30T16:15:15.244817855Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":215,"events_lines":2}
227
+ {"time":"2026-04-30T16:15:15.629764097Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
228
+ {"time":"2026-04-30T16:15:30.244419088Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":217,"events_lines":2}
229
+ {"time":"2026-04-30T16:15:30.718650456Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
230
+ {"time":"2026-04-30T16:15:45.244475433Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":219,"events_lines":2}
231
+ {"time":"2026-04-30T16:15:45.721065928Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
232
+ {"time":"2026-04-30T16:16:00.24427964Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":221,"events_lines":2}
233
+ {"time":"2026-04-30T16:16:00.621468603Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
234
+ {"time":"2026-04-30T16:16:15.244126051Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":223,"events_lines":2}
235
+ {"time":"2026-04-30T16:16:15.555099823Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
236
+ {"time":"2026-04-30T16:16:30.244817929Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":225,"events_lines":2}
237
+ {"time":"2026-04-30T16:16:30.566556009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
238
+ {"time":"2026-04-30T16:16:45.244192642Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":227,"events_lines":2}
239
+ {"time":"2026-04-30T16:16:45.542374436Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
240
+ {"time":"2026-04-30T16:17:00.245220123Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":229,"events_lines":2}
241
+ {"time":"2026-04-30T16:17:01.109944469Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
242
+ {"time":"2026-04-30T16:17:15.245002527Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":231,"events_lines":2}
243
+ {"time":"2026-04-30T16:17:15.611719009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
244
+ {"time":"2026-04-30T16:17:30.245053474Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":233,"events_lines":2}
245
+ {"time":"2026-04-30T16:17:30.562413801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
246
+ {"time":"2026-04-30T16:17:45.24513698Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":235,"events_lines":2}
247
+ {"time":"2026-04-30T16:17:45.491589211Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
248
+ {"time":"2026-04-30T16:18:00.244541037Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":237,"events_lines":2}
249
+ {"time":"2026-04-30T16:18:00.640114872Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
250
+ {"time":"2026-04-30T16:18:15.244559505Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":239,"events_lines":2}
251
+ {"time":"2026-04-30T16:18:15.845324763Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
252
+ {"time":"2026-04-30T16:18:30.244429902Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":241,"events_lines":2}
253
+ {"time":"2026-04-30T16:18:30.535722868Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
254
+ {"time":"2026-04-30T16:18:45.244604384Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":243,"events_lines":2}
255
+ {"time":"2026-04-30T16:18:45.624635597Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
256
+ {"time":"2026-04-30T16:19:00.244882812Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":245,"events_lines":2}
257
+ {"time":"2026-04-30T16:19:00.691855514Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
258
+ {"time":"2026-04-30T16:19:15.244745547Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":247,"events_lines":2,"console_offset":129,"console_lines":1}
259
+ {"time":"2026-04-30T16:19:15.619041231Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
260
+ {"time":"2026-04-30T16:19:30.245089032Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":249,"events_lines":2}
261
+ {"time":"2026-04-30T16:19:30.557376124Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
262
+ {"time":"2026-04-30T16:19:45.244706209Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":251,"events_lines":2}
263
+ {"time":"2026-04-30T16:19:45.487890371Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
264
+ {"time":"2026-04-30T16:20:00.244782624Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":253,"events_lines":2}
265
+ {"time":"2026-04-30T16:20:00.594540732Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
266
+ {"time":"2026-04-30T16:20:15.244804828Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":255,"events_lines":2}
267
+ {"time":"2026-04-30T16:20:15.594247223Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
268
+ {"time":"2026-04-30T16:20:30.245086571Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":257,"events_lines":2}
269
+ {"time":"2026-04-30T16:20:30.521426901Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
270
+ {"time":"2026-04-30T16:20:45.244179602Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":259,"events_lines":2}
271
+ {"time":"2026-04-30T16:20:45.646414563Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
272
+ {"time":"2026-04-30T16:21:00.244548215Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":261,"events_lines":2}
273
+ {"time":"2026-04-30T16:21:00.77872265Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
274
+ {"time":"2026-04-30T16:21:15.244696248Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":263,"events_lines":2}
275
+ {"time":"2026-04-30T16:21:15.642490972Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
276
+ {"time":"2026-04-30T16:21:30.245020274Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":265,"events_lines":2}
277
+ {"time":"2026-04-30T16:21:30.486987542Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
278
+ {"time":"2026-04-30T16:21:45.245262067Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":267,"events_lines":2}
279
+ {"time":"2026-04-30T16:21:45.567083803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
280
+ {"time":"2026-04-30T16:22:00.244661117Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":269,"events_lines":2}
281
+ {"time":"2026-04-30T16:22:02.030879918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
282
+ {"time":"2026-04-30T16:22:15.244640305Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":271,"events_lines":2}
283
+ {"time":"2026-04-30T16:22:15.822289437Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
284
+ {"time":"2026-04-30T16:22:30.24475682Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":273,"events_lines":2}
285
+ {"time":"2026-04-30T16:22:30.584061847Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
286
+ {"time":"2026-04-30T16:22:45.24470263Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":275,"events_lines":2}
287
+ {"time":"2026-04-30T16:22:45.65057256Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
288
+ {"time":"2026-04-30T16:23:00.244633587Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":277,"events_lines":2}
289
+ {"time":"2026-04-30T16:23:00.590020932Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
290
+ {"time":"2026-04-30T16:23:15.245000446Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":279,"events_lines":2}
291
+ {"time":"2026-04-30T16:23:15.789656475Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
292
+ {"time":"2026-04-30T16:23:30.244211676Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":281,"events_lines":2}
293
+ {"time":"2026-04-30T16:23:30.495419364Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
294
+ {"time":"2026-04-30T16:23:45.245165668Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":283,"events_lines":2}
295
+ {"time":"2026-04-30T16:23:46.021114787Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
296
+ {"time":"2026-04-30T16:24:00.244128274Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":285,"events_lines":2}
297
+ {"time":"2026-04-30T16:24:00.857859087Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
298
+ {"time":"2026-04-30T16:24:15.244173198Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":287,"events_lines":2}
299
+ {"time":"2026-04-30T16:24:15.546260259Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
300
+ {"time":"2026-04-30T16:24:30.244997175Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":289,"events_lines":2}
301
+ {"time":"2026-04-30T16:24:30.859929075Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
302
+ {"time":"2026-04-30T16:24:45.244626804Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":291,"events_lines":2}
303
+ {"time":"2026-04-30T16:24:45.805477914Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
304
+ {"time":"2026-04-30T16:25:00.245083838Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":293,"events_lines":2}
305
+ {"time":"2026-04-30T16:25:00.829768723Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
306
+ {"time":"2026-04-30T16:25:15.245179369Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":295,"events_lines":2}
307
+ {"time":"2026-04-30T16:25:15.651773092Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
308
+ {"time":"2026-04-30T16:25:30.24411927Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":297,"events_lines":2}
309
+ {"time":"2026-04-30T16:25:30.572123852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
310
+ {"time":"2026-04-30T16:25:45.244583916Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":299,"events_lines":2}
311
+ {"time":"2026-04-30T16:25:45.741505054Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
312
+ {"time":"2026-04-30T16:26:00.245036644Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":301,"events_lines":2}
313
+ {"time":"2026-04-30T16:26:00.570036154Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
314
+ {"time":"2026-04-30T16:26:15.24483666Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":303,"events_lines":2}
315
+ {"time":"2026-04-30T16:26:15.59237749Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
316
+ {"time":"2026-04-30T16:26:30.244309584Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":305,"events_lines":2}
317
+ {"time":"2026-04-30T16:26:30.520631523Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
318
+ {"time":"2026-04-30T16:26:45.244635591Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":307,"events_lines":2}
319
+ {"time":"2026-04-30T16:26:45.808559084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
320
+ {"time":"2026-04-30T16:27:00.24439155Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":309,"events_lines":2}
321
+ {"time":"2026-04-30T16:27:00.673589648Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
322
+ {"time":"2026-04-30T16:27:15.244605409Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":311,"events_lines":2}
323
+ {"time":"2026-04-30T16:27:15.603047182Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
324
+ {"time":"2026-04-30T16:27:30.244895273Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":313,"events_lines":2}
325
+ {"time":"2026-04-30T16:27:30.733862537Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
326
+ {"time":"2026-04-30T16:27:45.244596186Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":315,"events_lines":2}
327
+ {"time":"2026-04-30T16:27:45.551321503Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
328
+ {"time":"2026-04-30T16:28:00.244757638Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":317,"events_lines":2}
329
+ {"time":"2026-04-30T16:28:00.603616939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
330
+ {"time":"2026-04-30T16:28:15.244283227Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":319,"events_lines":2}
331
+ {"time":"2026-04-30T16:28:15.549705712Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
332
+ {"time":"2026-04-30T16:28:30.244169487Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":321,"events_lines":2}
333
+ {"time":"2026-04-30T16:28:30.583539623Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
334
+ {"time":"2026-04-30T16:28:45.244834295Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":323,"events_lines":2}
335
+ {"time":"2026-04-30T16:28:45.639240494Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
336
+ {"time":"2026-04-30T16:29:00.245018942Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":325,"events_lines":2}
337
+ {"time":"2026-04-30T16:29:00.575473332Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
338
+ {"time":"2026-04-30T16:29:15.245105913Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":327,"events_lines":2}
339
+ {"time":"2026-04-30T16:29:15.774910446Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
340
+ {"time":"2026-04-30T16:29:30.24484989Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":329,"events_lines":2}
341
+ {"time":"2026-04-30T16:29:30.526958565Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
342
+ {"time":"2026-04-30T16:29:45.245075449Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":331,"events_lines":2}
343
+ {"time":"2026-04-30T16:29:45.523227592Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
344
+ {"time":"2026-04-30T16:30:00.244503762Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":333,"events_lines":2}
345
+ {"time":"2026-04-30T16:30:00.60865092Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
346
+ {"time":"2026-04-30T16:30:15.244846146Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":335,"events_lines":2}
347
+ {"time":"2026-04-30T16:30:15.561865375Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
348
+ {"time":"2026-04-30T16:30:30.244647188Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":337,"events_lines":2}
349
+ {"time":"2026-04-30T16:30:31.040400261Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
350
+ {"time":"2026-04-30T16:30:45.244309382Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":339,"events_lines":2}
351
+ {"time":"2026-04-30T16:30:45.608934657Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
352
+ {"time":"2026-04-30T16:31:00.245018997Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":341,"events_lines":2}
353
+ {"time":"2026-04-30T16:31:00.561351857Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
354
+ {"time":"2026-04-30T16:31:15.244632888Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":343,"events_lines":2}
355
+ {"time":"2026-04-30T16:31:15.82265295Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
356
+ {"time":"2026-04-30T16:31:30.244311087Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":345,"events_lines":2}
357
+ {"time":"2026-04-30T16:31:30.615484037Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
358
+ {"time":"2026-04-30T16:31:45.244405409Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":347,"events_lines":2}
359
+ {"time":"2026-04-30T16:31:45.642912074Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
360
+ {"time":"2026-04-30T16:32:00.244587582Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":349,"events_lines":2}
361
+ {"time":"2026-04-30T16:32:00.851952585Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
362
+ {"time":"2026-04-30T16:32:15.245061654Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":351,"events_lines":2}
363
+ {"time":"2026-04-30T16:32:15.824309015Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
364
+ {"time":"2026-04-30T16:32:30.244137589Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":353,"events_lines":2}
365
+ {"time":"2026-04-30T16:32:30.808481519Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
366
+ {"time":"2026-04-30T16:32:45.24513166Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":355,"events_lines":2}
367
+ {"time":"2026-04-30T16:32:45.628953411Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
368
+ {"time":"2026-04-30T16:33:00.245013563Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":357,"events_lines":2}
369
+ {"time":"2026-04-30T16:33:00.551842125Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
370
+ {"time":"2026-04-30T16:33:15.244966543Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":359,"events_lines":2}
371
+ {"time":"2026-04-30T16:33:15.528391982Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
372
+ {"time":"2026-04-30T16:33:30.244773645Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":361,"events_lines":2}
373
+ {"time":"2026-04-30T16:33:30.479048873Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
374
+ {"time":"2026-04-30T16:33:45.244274136Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":363,"events_lines":2}
375
+ {"time":"2026-04-30T16:33:45.509706382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
376
+ {"time":"2026-04-30T16:34:00.24442273Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":365,"events_lines":2}
377
+ {"time":"2026-04-30T16:34:00.625618925Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
378
+ {"time":"2026-04-30T16:34:15.244205194Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":367,"events_lines":2}
379
+ {"time":"2026-04-30T16:34:15.590338398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
380
+ {"time":"2026-04-30T16:34:30.24467558Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":369,"events_lines":2}
381
+ {"time":"2026-04-30T16:34:30.611549656Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
382
+ {"time":"2026-04-30T16:34:45.24431518Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":371,"events_lines":2}
383
+ {"time":"2026-04-30T16:34:45.863881994Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
384
+ {"time":"2026-04-30T16:35:00.245054909Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":373,"events_lines":2}
385
+ {"time":"2026-04-30T16:35:00.714273148Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
386
+ {"time":"2026-04-30T16:35:15.24488778Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":375,"events_lines":2}
387
+ {"time":"2026-04-30T16:35:16.203423888Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
388
+ {"time":"2026-04-30T16:35:30.244242933Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":377,"events_lines":2}
389
+ {"time":"2026-04-30T16:35:30.525482556Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
390
+ {"time":"2026-04-30T16:35:45.244858967Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":379,"events_lines":2}
391
+ {"time":"2026-04-30T16:35:45.497267109Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
392
+ {"time":"2026-04-30T16:36:00.244652158Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":381,"events_lines":2}
393
+ {"time":"2026-04-30T16:36:00.601783609Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
394
+ {"time":"2026-04-30T16:36:15.244335945Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":383,"events_lines":2}
395
+ {"time":"2026-04-30T16:36:15.516555786Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
396
+ {"time":"2026-04-30T16:36:30.244410237Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":385,"events_lines":2}
397
+ {"time":"2026-04-30T16:36:30.507182415Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
398
+ {"time":"2026-04-30T16:36:45.244175665Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":387,"events_lines":2}
399
+ {"time":"2026-04-30T16:36:45.830948879Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
400
+ {"time":"2026-04-30T16:37:00.245044388Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":389,"events_lines":2}
401
+ {"time":"2026-04-30T16:37:00.587706919Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
402
+ {"time":"2026-04-30T16:37:15.244477539Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":391,"events_lines":2}
403
+ {"time":"2026-04-30T16:37:15.531008184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
404
+ {"time":"2026-04-30T16:37:30.244341578Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":393,"events_lines":2}
405
+ {"time":"2026-04-30T16:37:30.552069208Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
406
+ {"time":"2026-04-30T16:37:45.244447687Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":395,"events_lines":2}
407
+ {"time":"2026-04-30T16:37:45.838926472Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
408
+ {"time":"2026-04-30T16:38:00.244895419Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":397,"events_lines":2}
409
+ {"time":"2026-04-30T16:38:00.601529184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
410
+ {"time":"2026-04-30T16:38:15.245146516Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":399,"events_lines":2}
411
+ {"time":"2026-04-30T16:38:15.605067378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
412
+ {"time":"2026-04-30T16:38:30.245070162Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":401,"events_lines":2}
413
+ {"time":"2026-04-30T16:38:30.585471813Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
414
+ {"time":"2026-04-30T16:38:45.24473469Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":403,"events_lines":2}
415
+ {"time":"2026-04-30T16:38:45.585697943Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
416
+ {"time":"2026-04-30T16:39:00.244698917Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":405,"events_lines":2}
417
+ {"time":"2026-04-30T16:39:00.590288976Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
418
+ {"time":"2026-04-30T16:39:15.245083884Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":407,"events_lines":2,"console_offset":130,"console_lines":65}
419
+ {"time":"2026-04-30T16:39:15.579488395Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
420
+ {"time":"2026-04-30T16:39:30.244712047Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":409,"events_lines":2}
421
+ {"time":"2026-04-30T16:39:34.413897004Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
422
+ {"time":"2026-04-30T16:39:45.244882914Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":411,"events_lines":2}
423
+ {"time":"2026-04-30T16:39:45.549210818Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
424
+ {"time":"2026-04-30T16:40:00.24456604Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":413,"events_lines":2}
425
+ {"time":"2026-04-30T16:40:00.789163087Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
426
+ {"time":"2026-04-30T16:40:15.244683967Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":415,"events_lines":2}
427
+ {"time":"2026-04-30T16:40:15.797510992Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
428
+ {"time":"2026-04-30T16:40:29.698466699Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
429
+ {"time":"2026-04-30T16:40:29.698678598Z","level":"INFO","msg":"filestream: sending request","total_files":3,"history_offset":24,"history_lines":1,"events_offset":417,"events_lines":1,"uploaded_len":3,"complete":true,"exit_code":1}
430
+ {"time":"2026-04-30T16:40:29.949485773Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
431
+ {"time":"2026-04-30T16:40:29.949701778Z","level":"INFO","msg":"stream: finishing up"}
432
+ {"time":"2026-04-30T16:40:29.949715008Z","level":"INFO","msg":"handler: closed"}
433
+ {"time":"2026-04-30T16:40:29.951077549Z","level":"INFO","msg":"sender: closed"}
434
+ {"time":"2026-04-30T16:40:29.951086893Z","level":"INFO","msg":"stream: all finished"}
epfl_final/wandb/run-20260430_154759-yujhhjfq/logs/debug.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-04-30 15:47:59,264 INFO MainThread:220 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-04-30 15:47:59,264 INFO MainThread:220 [wandb_setup.py:_flush():81] Configure stats pid to 220
3
+ 2026-04-30 15:47:59,264 INFO MainThread:220 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-04-30 15:47:59,264 INFO MainThread:220 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_154759-yujhhjfq/logs/debug.log
5
+ 2026-04-30 15:47:59,265 INFO MainThread:220 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_154759-yujhhjfq/logs/debug-internal.log
6
+ 2026-04-30 15:47:59,265 INFO MainThread:220 [wandb_init.py:init():850] calling init triggers
7
+ 2026-04-30 15:47:59,265 INFO MainThread:220 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 3840, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_154756', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10005, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 9, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': True, 'sparse_layer_indices': [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27], '_wandb': {}}
9
+ 2026-04-30 15:47:59,265 INFO MainThread:220 [wandb_init.py:init():898] starting backend
10
+ 2026-04-30 15:47:59,497 INFO MainThread:220 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-04-30 15:47:59,796 INFO MainThread:220 [wandb_init.py:init():918] backend started and connected
12
+ 2026-04-30 15:47:59,798 INFO MainThread:220 [wandb_init.py:init():988] updated telemetry
13
+ 2026-04-30 15:47:59,799 INFO MainThread:220 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-04-30 15:48:00,182 INFO MainThread:220 [wandb_init.py:init():1056] starting run threads in backend
15
+ 2026-04-30 15:48:00,239 INFO MainThread:220 [wandb_run.py:_console_start():2554] atexit reg
16
+ 2026-04-30 15:48:00,239 INFO MainThread:220 [wandb_run.py:_redirect():2403] redirect: wrap_raw
17
+ 2026-04-30 15:48:00,239 INFO MainThread:220 [wandb_run.py:_redirect():2472] Wrapping output streams.
18
+ 2026-04-30 15:48:00,239 INFO MainThread:220 [wandb_run.py:_redirect():2495] Redirects installed.
19
+ 2026-04-30 15:48:00,242 INFO MainThread:220 [wandb_init.py:init():1094] run started, returning control to user process
20
+ 2026-04-30 16:40:29,211 INFO wandb-AsyncioManager-main:220 [service_client.py:_forward_responses():134] Reached EOF.
21
+ 2026-04-30 16:40:29,215 INFO wandb-AsyncioManager-main:220 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.