Hugging Face's logo

w-ahmad
/
STRESS-TEST2

Safetensors
Model card Files
xet
 Community
w-ahmad commited on
Commit
6381dfb
·
verified ·
1 Parent(s): 3b4810c

Auto upload zain 2026-08-14T21:14:29.212142 (part 4)

Browse files
Files changed (33) hide show
  1. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/config.json +36 -0
  2. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/model.safetensors +3 -0
  3. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/optimizer.pt +3 -0
  4. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/rng_state.pth +3 -0
  5. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/scheduler.pt +3 -0
  6. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/tokenizer.json +0 -0
  7. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/tokenizer_config.json +13 -0
  8. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/trainer_state.json +279 -0
  9. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/training_args.bin +3 -0
  10. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/config.json +36 -0
  11. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/model.safetensors +3 -0
  12. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/optimizer.pt +3 -0
  13. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/rng_state.pth +3 -0
  14. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/scheduler.pt +3 -0
  15. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/tokenizer.json +0 -0
  16. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/tokenizer_config.json +13 -0
  17. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/trainer_state.json +314 -0
  18. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/training_args.bin +3 -0
  19. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/config.json +36 -0
  20. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/model.safetensors +3 -0
  21. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/optimizer.pt +3 -0
  22. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/rng_state.pth +3 -0
  23. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/scheduler.pt +3 -0
  24. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/tokenizer.json +0 -0
  25. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/tokenizer_config.json +13 -0
  26. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/trainer_state.json +349 -0
  27. zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/training_args.bin +3 -0
  28. zain/Activation/out/glu-situglu_low-9L_run/config.json +36 -0
  29. zain/Activation/out/glu-situglu_low-9L_run/model.safetensors +3 -0
  30. zain/Activation/out/glu-situglu_low-9L_run/tokenizer.json +0 -0
  31. zain/Activation/out/glu-situglu_low-9L_run/tokenizer_config.json +13 -0
  32. zain/Activation/out/glu-situglu_low-9L_run/training_args.bin +3 -0
  33. zain/Activation/out/glu-situglu_low-9L_run/training_log.jsonl +22 -0
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "situglu_low",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "glu",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7293df7d86ad778a5423ed5dd02d811d71347e6011f98fb419fb8cbbb0ab09ea
3
+ size 4011496
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5c8667c0431c49e3a6d47f118f557f6b4baebcfc350b02f80191952621fed5a
3
+ size 8074746
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
3
+ size 14244
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3173cd6908322b02fdacbfeb79bbb6a1a1a40d218262cceb79ee8c11c4fa07d7
3
+ size 1064
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/trainer_state.json ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.37735849056603776,
6
+ "eval_steps": 60000,
7
+ "global_step": 700,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9765625,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.168720245361328,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.349609375,
22
+ "learning_rate": 0.001,
23
+ "loss": 6.01421890258789,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.03234501347708895,
28
+ "grad_norm": 1.0703125,
29
+ "learning_rate": 0.001,
30
+ "loss": 5.568880081176758,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.0431266846361186,
35
+ "grad_norm": 0.5078125,
36
+ "learning_rate": 0.001,
37
+ "loss": 5.037441253662109,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.05390835579514825,
42
+ "grad_norm": 0.451171875,
43
+ "learning_rate": 0.001,
44
+ "loss": 4.574560165405273,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.0646900269541779,
49
+ "grad_norm": 0.392578125,
50
+ "learning_rate": 0.001,
51
+ "loss": 4.265085601806641,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.07547169811320754,
56
+ "grad_norm": 0.84765625,
57
+ "learning_rate": 0.001,
58
+ "loss": 4.019483184814453,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.0862533692722372,
63
+ "grad_norm": 0.828125,
64
+ "learning_rate": 0.001,
65
+ "loss": 3.814248275756836,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.09703504043126684,
70
+ "grad_norm": 1.0,
71
+ "learning_rate": 0.001,
72
+ "loss": 3.6563358306884766,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.1078167115902965,
77
+ "grad_norm": 0.875,
78
+ "learning_rate": 0.001,
79
+ "loss": 3.519593048095703,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.11859838274932614,
84
+ "grad_norm": 0.57421875,
85
+ "learning_rate": 0.001,
86
+ "loss": 3.401234436035156,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.1293800539083558,
91
+ "grad_norm": 0.5,
92
+ "learning_rate": 0.001,
93
+ "loss": 3.3191593170166014,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.14016172506738545,
98
+ "grad_norm": 0.78125,
99
+ "learning_rate": 0.001,
100
+ "loss": 3.2113693237304686,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.1509433962264151,
105
+ "grad_norm": 0.6640625,
106
+ "learning_rate": 0.001,
107
+ "loss": 3.125957489013672,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.16172506738544473,
112
+ "grad_norm": 1.0703125,
113
+ "learning_rate": 0.001,
114
+ "loss": 3.0545766830444334,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.1725067385444744,
119
+ "grad_norm": 0.458984375,
120
+ "learning_rate": 0.001,
121
+ "loss": 2.979182815551758,
122
+ "step": 320
123
+ },
124
+ {
125
+ "epoch": 0.18328840970350405,
126
+ "grad_norm": 0.8828125,
127
+ "learning_rate": 0.001,
128
+ "loss": 2.92999153137207,
129
+ "step": 340
130
+ },
131
+ {
132
+ "epoch": 0.1940700808625337,
133
+ "grad_norm": 0.51171875,
134
+ "learning_rate": 0.001,
135
+ "loss": 2.867526626586914,
136
+ "step": 360
137
+ },
138
+ {
139
+ "epoch": 0.20485175202156333,
140
+ "grad_norm": 0.6328125,
141
+ "learning_rate": 0.001,
142
+ "loss": 2.8189437866210936,
143
+ "step": 380
144
+ },
145
+ {
146
+ "epoch": 0.215633423180593,
147
+ "grad_norm": 0.66796875,
148
+ "learning_rate": 0.001,
149
+ "loss": 2.773433303833008,
150
+ "step": 400
151
+ },
152
+ {
153
+ "epoch": 0.22641509433962265,
154
+ "grad_norm": 0.62890625,
155
+ "learning_rate": 0.001,
156
+ "loss": 2.7344507217407226,
157
+ "step": 420
158
+ },
159
+ {
160
+ "epoch": 0.2371967654986523,
161
+ "grad_norm": 0.71875,
162
+ "learning_rate": 0.001,
163
+ "loss": 2.710928726196289,
164
+ "step": 440
165
+ },
166
+ {
167
+ "epoch": 0.24797843665768193,
168
+ "grad_norm": 0.69921875,
169
+ "learning_rate": 0.001,
170
+ "loss": 2.6710628509521483,
171
+ "step": 460
172
+ },
173
+ {
174
+ "epoch": 0.2587601078167116,
175
+ "grad_norm": 0.62109375,
176
+ "learning_rate": 0.001,
177
+ "loss": 2.6398643493652343,
178
+ "step": 480
179
+ },
180
+ {
181
+ "epoch": 0.2695417789757412,
182
+ "grad_norm": 0.54296875,
183
+ "learning_rate": 0.001,
184
+ "loss": 2.6106098175048826,
185
+ "step": 500
186
+ },
187
+ {
188
+ "epoch": 0.2803234501347709,
189
+ "grad_norm": 0.84375,
190
+ "learning_rate": 0.001,
191
+ "loss": 2.587799072265625,
192
+ "step": 520
193
+ },
194
+ {
195
+ "epoch": 0.29110512129380056,
196
+ "grad_norm": 0.6015625,
197
+ "learning_rate": 0.001,
198
+ "loss": 2.5707231521606446,
199
+ "step": 540
200
+ },
201
+ {
202
+ "epoch": 0.3018867924528302,
203
+ "grad_norm": 0.6171875,
204
+ "learning_rate": 0.001,
205
+ "loss": 2.545253944396973,
206
+ "step": 560
207
+ },
208
+ {
209
+ "epoch": 0.31266846361185985,
210
+ "grad_norm": 0.57421875,
211
+ "learning_rate": 0.001,
212
+ "loss": 2.5261125564575195,
213
+ "step": 580
214
+ },
215
+ {
216
+ "epoch": 0.32345013477088946,
217
+ "grad_norm": 0.96875,
218
+ "learning_rate": 0.001,
219
+ "loss": 2.503111457824707,
220
+ "step": 600
221
+ },
222
+ {
223
+ "epoch": 0.33423180592991913,
224
+ "grad_norm": 0.59375,
225
+ "learning_rate": 0.001,
226
+ "loss": 2.4910961151123048,
227
+ "step": 620
228
+ },
229
+ {
230
+ "epoch": 0.3450134770889488,
231
+ "grad_norm": 0.51953125,
232
+ "learning_rate": 0.001,
233
+ "loss": 2.4673994064331053,
234
+ "step": 640
235
+ },
236
+ {
237
+ "epoch": 0.3557951482479784,
238
+ "grad_norm": 0.7265625,
239
+ "learning_rate": 0.001,
240
+ "loss": 2.4556552886962892,
241
+ "step": 660
242
+ },
243
+ {
244
+ "epoch": 0.3665768194070081,
245
+ "grad_norm": 0.9609375,
246
+ "learning_rate": 0.001,
247
+ "loss": 2.438701057434082,
248
+ "step": 680
249
+ },
250
+ {
251
+ "epoch": 0.37735849056603776,
252
+ "grad_norm": 0.73828125,
253
+ "learning_rate": 0.001,
254
+ "loss": 2.4236507415771484,
255
+ "step": 700
256
+ }
257
+ ],
258
+ "logging_steps": 20,
259
+ "max_steps": 1000,
260
+ "num_input_tokens_seen": 0,
261
+ "num_train_epochs": 1,
262
+ "save_steps": 100,
263
+ "stateful_callbacks": {
264
+ "TrainerControl": {
265
+ "args": {
266
+ "should_epoch_stop": false,
267
+ "should_evaluate": false,
268
+ "should_log": false,
269
+ "should_save": true,
270
+ "should_training_stop": false
271
+ },
272
+ "attributes": {}
273
+ }
274
+ },
275
+ "total_flos": 1626175281561600.0,
276
+ "train_batch_size": 512,
277
+ "trial_name": null,
278
+ "trial_params": null
279
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-700/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f25d0bd9783b93f4f41ff419d306235c223c206b24ccc727fc5a1503e39e36ba
3
+ size 4920
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "situglu_low",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "glu",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8b4e43464808549b9dddd99ec8688d796f75bd273d33bdbff14a631de091912
3
+ size 4011496
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f32e5db64cdb073a2300fa489658cd89ae6fa4ff6c6f279d2af51f34500415d6
3
+ size 8074746
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
3
+ size 14244
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7334026bbd250befa466dcbe6bec8bfe84c03024263e4dea55e40698f2caa92e
3
+ size 1064
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/trainer_state.json ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.431266846361186,
6
+ "eval_steps": 60000,
7
+ "global_step": 800,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9765625,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.168720245361328,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.349609375,
22
+ "learning_rate": 0.001,
23
+ "loss": 6.01421890258789,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.03234501347708895,
28
+ "grad_norm": 1.0703125,
29
+ "learning_rate": 0.001,
30
+ "loss": 5.568880081176758,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.0431266846361186,
35
+ "grad_norm": 0.5078125,
36
+ "learning_rate": 0.001,
37
+ "loss": 5.037441253662109,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.05390835579514825,
42
+ "grad_norm": 0.451171875,
43
+ "learning_rate": 0.001,
44
+ "loss": 4.574560165405273,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.0646900269541779,
49
+ "grad_norm": 0.392578125,
50
+ "learning_rate": 0.001,
51
+ "loss": 4.265085601806641,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.07547169811320754,
56
+ "grad_norm": 0.84765625,
57
+ "learning_rate": 0.001,
58
+ "loss": 4.019483184814453,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.0862533692722372,
63
+ "grad_norm": 0.828125,
64
+ "learning_rate": 0.001,
65
+ "loss": 3.814248275756836,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.09703504043126684,
70
+ "grad_norm": 1.0,
71
+ "learning_rate": 0.001,
72
+ "loss": 3.6563358306884766,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.1078167115902965,
77
+ "grad_norm": 0.875,
78
+ "learning_rate": 0.001,
79
+ "loss": 3.519593048095703,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.11859838274932614,
84
+ "grad_norm": 0.57421875,
85
+ "learning_rate": 0.001,
86
+ "loss": 3.401234436035156,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.1293800539083558,
91
+ "grad_norm": 0.5,
92
+ "learning_rate": 0.001,
93
+ "loss": 3.3191593170166014,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.14016172506738545,
98
+ "grad_norm": 0.78125,
99
+ "learning_rate": 0.001,
100
+ "loss": 3.2113693237304686,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.1509433962264151,
105
+ "grad_norm": 0.6640625,
106
+ "learning_rate": 0.001,
107
+ "loss": 3.125957489013672,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.16172506738544473,
112
+ "grad_norm": 1.0703125,
113
+ "learning_rate": 0.001,
114
+ "loss": 3.0545766830444334,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.1725067385444744,
119
+ "grad_norm": 0.458984375,
120
+ "learning_rate": 0.001,
121
+ "loss": 2.979182815551758,
122
+ "step": 320
123
+ },
124
+ {
125
+ "epoch": 0.18328840970350405,
126
+ "grad_norm": 0.8828125,
127
+ "learning_rate": 0.001,
128
+ "loss": 2.92999153137207,
129
+ "step": 340
130
+ },
131
+ {
132
+ "epoch": 0.1940700808625337,
133
+ "grad_norm": 0.51171875,
134
+ "learning_rate": 0.001,
135
+ "loss": 2.867526626586914,
136
+ "step": 360
137
+ },
138
+ {
139
+ "epoch": 0.20485175202156333,
140
+ "grad_norm": 0.6328125,
141
+ "learning_rate": 0.001,
142
+ "loss": 2.8189437866210936,
143
+ "step": 380
144
+ },
145
+ {
146
+ "epoch": 0.215633423180593,
147
+ "grad_norm": 0.66796875,
148
+ "learning_rate": 0.001,
149
+ "loss": 2.773433303833008,
150
+ "step": 400
151
+ },
152
+ {
153
+ "epoch": 0.22641509433962265,
154
+ "grad_norm": 0.62890625,
155
+ "learning_rate": 0.001,
156
+ "loss": 2.7344507217407226,
157
+ "step": 420
158
+ },
159
+ {
160
+ "epoch": 0.2371967654986523,
161
+ "grad_norm": 0.71875,
162
+ "learning_rate": 0.001,
163
+ "loss": 2.710928726196289,
164
+ "step": 440
165
+ },
166
+ {
167
+ "epoch": 0.24797843665768193,
168
+ "grad_norm": 0.69921875,
169
+ "learning_rate": 0.001,
170
+ "loss": 2.6710628509521483,
171
+ "step": 460
172
+ },
173
+ {
174
+ "epoch": 0.2587601078167116,
175
+ "grad_norm": 0.62109375,
176
+ "learning_rate": 0.001,
177
+ "loss": 2.6398643493652343,
178
+ "step": 480
179
+ },
180
+ {
181
+ "epoch": 0.2695417789757412,
182
+ "grad_norm": 0.54296875,
183
+ "learning_rate": 0.001,
184
+ "loss": 2.6106098175048826,
185
+ "step": 500
186
+ },
187
+ {
188
+ "epoch": 0.2803234501347709,
189
+ "grad_norm": 0.84375,
190
+ "learning_rate": 0.001,
191
+ "loss": 2.587799072265625,
192
+ "step": 520
193
+ },
194
+ {
195
+ "epoch": 0.29110512129380056,
196
+ "grad_norm": 0.6015625,
197
+ "learning_rate": 0.001,
198
+ "loss": 2.5707231521606446,
199
+ "step": 540
200
+ },
201
+ {
202
+ "epoch": 0.3018867924528302,
203
+ "grad_norm": 0.6171875,
204
+ "learning_rate": 0.001,
205
+ "loss": 2.545253944396973,
206
+ "step": 560
207
+ },
208
+ {
209
+ "epoch": 0.31266846361185985,
210
+ "grad_norm": 0.57421875,
211
+ "learning_rate": 0.001,
212
+ "loss": 2.5261125564575195,
213
+ "step": 580
214
+ },
215
+ {
216
+ "epoch": 0.32345013477088946,
217
+ "grad_norm": 0.96875,
218
+ "learning_rate": 0.001,
219
+ "loss": 2.503111457824707,
220
+ "step": 600
221
+ },
222
+ {
223
+ "epoch": 0.33423180592991913,
224
+ "grad_norm": 0.59375,
225
+ "learning_rate": 0.001,
226
+ "loss": 2.4910961151123048,
227
+ "step": 620
228
+ },
229
+ {
230
+ "epoch": 0.3450134770889488,
231
+ "grad_norm": 0.51953125,
232
+ "learning_rate": 0.001,
233
+ "loss": 2.4673994064331053,
234
+ "step": 640
235
+ },
236
+ {
237
+ "epoch": 0.3557951482479784,
238
+ "grad_norm": 0.7265625,
239
+ "learning_rate": 0.001,
240
+ "loss": 2.4556552886962892,
241
+ "step": 660
242
+ },
243
+ {
244
+ "epoch": 0.3665768194070081,
245
+ "grad_norm": 0.9609375,
246
+ "learning_rate": 0.001,
247
+ "loss": 2.438701057434082,
248
+ "step": 680
249
+ },
250
+ {
251
+ "epoch": 0.37735849056603776,
252
+ "grad_norm": 0.73828125,
253
+ "learning_rate": 0.001,
254
+ "loss": 2.4236507415771484,
255
+ "step": 700
256
+ },
257
+ {
258
+ "epoch": 0.3881401617250674,
259
+ "grad_norm": 0.59375,
260
+ "learning_rate": 0.001,
261
+ "loss": 2.4043102264404297,
262
+ "step": 720
263
+ },
264
+ {
265
+ "epoch": 0.39892183288409705,
266
+ "grad_norm": 0.6171875,
267
+ "learning_rate": 0.001,
268
+ "loss": 2.3941486358642576,
269
+ "step": 740
270
+ },
271
+ {
272
+ "epoch": 0.40970350404312667,
273
+ "grad_norm": 0.76171875,
274
+ "learning_rate": 0.001,
275
+ "loss": 2.3835357666015624,
276
+ "step": 760
277
+ },
278
+ {
279
+ "epoch": 0.42048517520215634,
280
+ "grad_norm": 0.58203125,
281
+ "learning_rate": 0.001,
282
+ "loss": 2.3709957122802736,
283
+ "step": 780
284
+ },
285
+ {
286
+ "epoch": 0.431266846361186,
287
+ "grad_norm": 0.59375,
288
+ "learning_rate": 0.001,
289
+ "loss": 2.352398490905762,
290
+ "step": 800
291
+ }
292
+ ],
293
+ "logging_steps": 20,
294
+ "max_steps": 1000,
295
+ "num_input_tokens_seen": 0,
296
+ "num_train_epochs": 1,
297
+ "save_steps": 100,
298
+ "stateful_callbacks": {
299
+ "TrainerControl": {
300
+ "args": {
301
+ "should_epoch_stop": false,
302
+ "should_evaluate": false,
303
+ "should_log": false,
304
+ "should_save": true,
305
+ "should_training_stop": false
306
+ },
307
+ "attributes": {}
308
+ }
309
+ },
310
+ "total_flos": 1858486036070400.0,
311
+ "train_batch_size": 512,
312
+ "trial_name": null,
313
+ "trial_params": null
314
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-800/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f25d0bd9783b93f4f41ff419d306235c223c206b24ccc727fc5a1503e39e36ba
3
+ size 4920
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "situglu_low",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "glu",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e14093bb337e274b84d88c1b3e07ffda5407ded2929358e6c10610b8fea7969
3
+ size 4011496
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00e35a052ef6c5a72ac3e3851b9c299ba993cece5764310632a097d6db908370
3
+ size 8074746
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
3
+ size 14244
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d48b00fea613f19335b87fb622648d28d7f8900963d07ba43c75df2b4dc3d4e
3
+ size 1064
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/trainer_state.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.48517520215633425,
6
+ "eval_steps": 60000,
7
+ "global_step": 900,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9765625,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.168720245361328,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.349609375,
22
+ "learning_rate": 0.001,
23
+ "loss": 6.01421890258789,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.03234501347708895,
28
+ "grad_norm": 1.0703125,
29
+ "learning_rate": 0.001,
30
+ "loss": 5.568880081176758,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.0431266846361186,
35
+ "grad_norm": 0.5078125,
36
+ "learning_rate": 0.001,
37
+ "loss": 5.037441253662109,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.05390835579514825,
42
+ "grad_norm": 0.451171875,
43
+ "learning_rate": 0.001,
44
+ "loss": 4.574560165405273,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.0646900269541779,
49
+ "grad_norm": 0.392578125,
50
+ "learning_rate": 0.001,
51
+ "loss": 4.265085601806641,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.07547169811320754,
56
+ "grad_norm": 0.84765625,
57
+ "learning_rate": 0.001,
58
+ "loss": 4.019483184814453,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.0862533692722372,
63
+ "grad_norm": 0.828125,
64
+ "learning_rate": 0.001,
65
+ "loss": 3.814248275756836,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.09703504043126684,
70
+ "grad_norm": 1.0,
71
+ "learning_rate": 0.001,
72
+ "loss": 3.6563358306884766,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.1078167115902965,
77
+ "grad_norm": 0.875,
78
+ "learning_rate": 0.001,
79
+ "loss": 3.519593048095703,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.11859838274932614,
84
+ "grad_norm": 0.57421875,
85
+ "learning_rate": 0.001,
86
+ "loss": 3.401234436035156,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.1293800539083558,
91
+ "grad_norm": 0.5,
92
+ "learning_rate": 0.001,
93
+ "loss": 3.3191593170166014,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.14016172506738545,
98
+ "grad_norm": 0.78125,
99
+ "learning_rate": 0.001,
100
+ "loss": 3.2113693237304686,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.1509433962264151,
105
+ "grad_norm": 0.6640625,
106
+ "learning_rate": 0.001,
107
+ "loss": 3.125957489013672,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.16172506738544473,
112
+ "grad_norm": 1.0703125,
113
+ "learning_rate": 0.001,
114
+ "loss": 3.0545766830444334,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.1725067385444744,
119
+ "grad_norm": 0.458984375,
120
+ "learning_rate": 0.001,
121
+ "loss": 2.979182815551758,
122
+ "step": 320
123
+ },
124
+ {
125
+ "epoch": 0.18328840970350405,
126
+ "grad_norm": 0.8828125,
127
+ "learning_rate": 0.001,
128
+ "loss": 2.92999153137207,
129
+ "step": 340
130
+ },
131
+ {
132
+ "epoch": 0.1940700808625337,
133
+ "grad_norm": 0.51171875,
134
+ "learning_rate": 0.001,
135
+ "loss": 2.867526626586914,
136
+ "step": 360
137
+ },
138
+ {
139
+ "epoch": 0.20485175202156333,
140
+ "grad_norm": 0.6328125,
141
+ "learning_rate": 0.001,
142
+ "loss": 2.8189437866210936,
143
+ "step": 380
144
+ },
145
+ {
146
+ "epoch": 0.215633423180593,
147
+ "grad_norm": 0.66796875,
148
+ "learning_rate": 0.001,
149
+ "loss": 2.773433303833008,
150
+ "step": 400
151
+ },
152
+ {
153
+ "epoch": 0.22641509433962265,
154
+ "grad_norm": 0.62890625,
155
+ "learning_rate": 0.001,
156
+ "loss": 2.7344507217407226,
157
+ "step": 420
158
+ },
159
+ {
160
+ "epoch": 0.2371967654986523,
161
+ "grad_norm": 0.71875,
162
+ "learning_rate": 0.001,
163
+ "loss": 2.710928726196289,
164
+ "step": 440
165
+ },
166
+ {
167
+ "epoch": 0.24797843665768193,
168
+ "grad_norm": 0.69921875,
169
+ "learning_rate": 0.001,
170
+ "loss": 2.6710628509521483,
171
+ "step": 460
172
+ },
173
+ {
174
+ "epoch": 0.2587601078167116,
175
+ "grad_norm": 0.62109375,
176
+ "learning_rate": 0.001,
177
+ "loss": 2.6398643493652343,
178
+ "step": 480
179
+ },
180
+ {
181
+ "epoch": 0.2695417789757412,
182
+ "grad_norm": 0.54296875,
183
+ "learning_rate": 0.001,
184
+ "loss": 2.6106098175048826,
185
+ "step": 500
186
+ },
187
+ {
188
+ "epoch": 0.2803234501347709,
189
+ "grad_norm": 0.84375,
190
+ "learning_rate": 0.001,
191
+ "loss": 2.587799072265625,
192
+ "step": 520
193
+ },
194
+ {
195
+ "epoch": 0.29110512129380056,
196
+ "grad_norm": 0.6015625,
197
+ "learning_rate": 0.001,
198
+ "loss": 2.5707231521606446,
199
+ "step": 540
200
+ },
201
+ {
202
+ "epoch": 0.3018867924528302,
203
+ "grad_norm": 0.6171875,
204
+ "learning_rate": 0.001,
205
+ "loss": 2.545253944396973,
206
+ "step": 560
207
+ },
208
+ {
209
+ "epoch": 0.31266846361185985,
210
+ "grad_norm": 0.57421875,
211
+ "learning_rate": 0.001,
212
+ "loss": 2.5261125564575195,
213
+ "step": 580
214
+ },
215
+ {
216
+ "epoch": 0.32345013477088946,
217
+ "grad_norm": 0.96875,
218
+ "learning_rate": 0.001,
219
+ "loss": 2.503111457824707,
220
+ "step": 600
221
+ },
222
+ {
223
+ "epoch": 0.33423180592991913,
224
+ "grad_norm": 0.59375,
225
+ "learning_rate": 0.001,
226
+ "loss": 2.4910961151123048,
227
+ "step": 620
228
+ },
229
+ {
230
+ "epoch": 0.3450134770889488,
231
+ "grad_norm": 0.51953125,
232
+ "learning_rate": 0.001,
233
+ "loss": 2.4673994064331053,
234
+ "step": 640
235
+ },
236
+ {
237
+ "epoch": 0.3557951482479784,
238
+ "grad_norm": 0.7265625,
239
+ "learning_rate": 0.001,
240
+ "loss": 2.4556552886962892,
241
+ "step": 660
242
+ },
243
+ {
244
+ "epoch": 0.3665768194070081,
245
+ "grad_norm": 0.9609375,
246
+ "learning_rate": 0.001,
247
+ "loss": 2.438701057434082,
248
+ "step": 680
249
+ },
250
+ {
251
+ "epoch": 0.37735849056603776,
252
+ "grad_norm": 0.73828125,
253
+ "learning_rate": 0.001,
254
+ "loss": 2.4236507415771484,
255
+ "step": 700
256
+ },
257
+ {
258
+ "epoch": 0.3881401617250674,
259
+ "grad_norm": 0.59375,
260
+ "learning_rate": 0.001,
261
+ "loss": 2.4043102264404297,
262
+ "step": 720
263
+ },
264
+ {
265
+ "epoch": 0.39892183288409705,
266
+ "grad_norm": 0.6171875,
267
+ "learning_rate": 0.001,
268
+ "loss": 2.3941486358642576,
269
+ "step": 740
270
+ },
271
+ {
272
+ "epoch": 0.40970350404312667,
273
+ "grad_norm": 0.76171875,
274
+ "learning_rate": 0.001,
275
+ "loss": 2.3835357666015624,
276
+ "step": 760
277
+ },
278
+ {
279
+ "epoch": 0.42048517520215634,
280
+ "grad_norm": 0.58203125,
281
+ "learning_rate": 0.001,
282
+ "loss": 2.3709957122802736,
283
+ "step": 780
284
+ },
285
+ {
286
+ "epoch": 0.431266846361186,
287
+ "grad_norm": 0.59375,
288
+ "learning_rate": 0.001,
289
+ "loss": 2.352398490905762,
290
+ "step": 800
291
+ },
292
+ {
293
+ "epoch": 0.4420485175202156,
294
+ "grad_norm": 0.73046875,
295
+ "learning_rate": 0.001,
296
+ "loss": 2.3425289154052735,
297
+ "step": 820
298
+ },
299
+ {
300
+ "epoch": 0.4528301886792453,
301
+ "grad_norm": 0.66796875,
302
+ "learning_rate": 0.001,
303
+ "loss": 2.3336166381835937,
304
+ "step": 840
305
+ },
306
+ {
307
+ "epoch": 0.4636118598382749,
308
+ "grad_norm": 0.765625,
309
+ "learning_rate": 0.001,
310
+ "loss": 2.3204221725463867,
311
+ "step": 860
312
+ },
313
+ {
314
+ "epoch": 0.4743935309973046,
315
+ "grad_norm": 0.69921875,
316
+ "learning_rate": 0.001,
317
+ "loss": 2.3104717254638674,
318
+ "step": 880
319
+ },
320
+ {
321
+ "epoch": 0.48517520215633425,
322
+ "grad_norm": 0.73828125,
323
+ "learning_rate": 0.001,
324
+ "loss": 2.300767707824707,
325
+ "step": 900
326
+ }
327
+ ],
328
+ "logging_steps": 20,
329
+ "max_steps": 1000,
330
+ "num_input_tokens_seen": 0,
331
+ "num_train_epochs": 1,
332
+ "save_steps": 100,
333
+ "stateful_callbacks": {
334
+ "TrainerControl": {
335
+ "args": {
336
+ "should_epoch_stop": false,
337
+ "should_evaluate": false,
338
+ "should_log": false,
339
+ "should_save": true,
340
+ "should_training_stop": false
341
+ },
342
+ "attributes": {}
343
+ }
344
+ },
345
+ "total_flos": 2090796790579200.0,
346
+ "train_batch_size": 512,
347
+ "trial_name": null,
348
+ "trial_params": null
349
+ }
zain/Activation/out/glu-situglu_low-9L_run/checkpoint-900/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f25d0bd9783b93f4f41ff419d306235c223c206b24ccc727fc5a1503e39e36ba
3
+ size 4920
zain/Activation/out/glu-situglu_low-9L_run/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "situglu_low",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "glu",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/glu-situglu_low-9L_run/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e233ee80a9271c0f7245966ae8c5e63e06504a7f3b8db3384bfc2f7ea67ca352
3
+ size 4011496
zain/Activation/out/glu-situglu_low-9L_run/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/glu-situglu_low-9L_run/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/glu-situglu_low-9L_run/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f25d0bd9783b93f4f41ff419d306235c223c206b24ccc727fc5a1503e39e36ba
3
+ size 4920
zain/Activation/out/glu-situglu_low-9L_run/training_log.jsonl CHANGED
@@ -29,3 +29,25 @@
29
  {"step": 580, "epoch": 0.31266846361185985, "timestamp": 1786741724.5763056, "loss": 2.5261125564575195, "grad_norm": 0.57421875, "learning_rate": 0.001, "train/total_time_seconds": 188.5669750533998, "train/time_per_step_avg": 0.2774039166048169, "train/epoch_time_elapsed": 407.9640324935317, "train/estimated_remaining_minutes": 2.2758083196099976}
30
  {"step": 600, "epoch": 0.32345013477088946, "timestamp": 1786741738.8271275, "loss": 2.503111457824707, "grad_norm": 0.96875, "learning_rate": 0.001, "train/total_time_seconds": 195.11553824692965, "train/time_per_step_avg": 0.2860154477879405, "train/epoch_time_elapsed": 422.2148540392518, "train/estimated_remaining_minutes": 2.1679504249658854}
31
  {"step": 620, "epoch": 0.33423180592991913, "timestamp": 1786741753.028786, "loss": 2.4910961151123048, "grad_norm": 0.59375, "learning_rate": 0.001, "train/total_time_seconds": 201.70382872596383, "train/time_per_step_avg": 0.31366395972669125, "train/epoch_time_elapsed": 436.4165124334395, "train/estimated_remaining_minutes": 2.0604154547275875}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  {"step": 580, "epoch": 0.31266846361185985, "timestamp": 1786741724.5763056, "loss": 2.5261125564575195, "grad_norm": 0.57421875, "learning_rate": 0.001, "train/total_time_seconds": 188.5669750533998, "train/time_per_step_avg": 0.2774039166048169, "train/epoch_time_elapsed": 407.9640324935317, "train/estimated_remaining_minutes": 2.2758083196099976}
30
  {"step": 600, "epoch": 0.32345013477088946, "timestamp": 1786741738.8271275, "loss": 2.503111457824707, "grad_norm": 0.96875, "learning_rate": 0.001, "train/total_time_seconds": 195.11553824692965, "train/time_per_step_avg": 0.2860154477879405, "train/epoch_time_elapsed": 422.2148540392518, "train/estimated_remaining_minutes": 2.1679504249658854}
31
  {"step": 620, "epoch": 0.33423180592991913, "timestamp": 1786741753.028786, "loss": 2.4910961151123048, "grad_norm": 0.59375, "learning_rate": 0.001, "train/total_time_seconds": 201.70382872596383, "train/time_per_step_avg": 0.31366395972669125, "train/epoch_time_elapsed": 436.4165124334395, "train/estimated_remaining_minutes": 2.0604154547275875}
32
+ {"step": 640, "epoch": 0.3450134770889488, "timestamp": 1786741767.4623919, "loss": 2.4673994064331053, "grad_norm": 0.51953125, "learning_rate": 0.001, "train/total_time_seconds": 208.4073604159057, "train/time_per_step_avg": 0.330281013995409, "train/epoch_time_elapsed": 450.8501185141504, "train/estimated_remaining_minutes": 1.9538190038991161}
33
+ {"step": 660, "epoch": 0.3557951482479784, "timestamp": 1786741782.213134, "loss": 2.4556552886962892, "grad_norm": 0.7265625, "learning_rate": 0.001, "train/total_time_seconds": 215.17041959613562, "train/time_per_step_avg": 0.33485721368342636, "train/epoch_time_elapsed": 465.6008605621755, "train/estimated_remaining_minutes": 1.8474227945122754}
34
+ {"step": 680, "epoch": 0.3665768194070081, "timestamp": 1786741796.9049735, "loss": 2.438701057434082, "grad_norm": 0.9609375, "learning_rate": 0.001, "train/total_time_seconds": 222.12999976053834, "train/time_per_step_avg": 0.3356302470713854, "train/epoch_time_elapsed": 480.29270008578897, "train/estimated_remaining_minutes": 1.7421960765532418}
35
+ {"step": 700, "epoch": 0.37735849056603776, "timestamp": 1786741811.3127532, "loss": 2.4236507415771484, "grad_norm": 0.73828125, "learning_rate": 0.001, "train/total_time_seconds": 228.83179549500346, "train/time_per_step_avg": 0.33716257248073817, "train/epoch_time_elapsed": 494.70047976076603, "train/estimated_remaining_minutes": 1.6345128249643104}
36
+ {"step": 720, "epoch": 0.3881401617250674, "timestamp": 1786741825.5428483, "loss": 2.4043102264404297, "grad_norm": 0.59375, "learning_rate": 0.001, "train/total_time_seconds": 235.36393900588155, "train/time_per_step_avg": 0.33660110279917715, "train/epoch_time_elapsed": 508.93057506531477, "train/estimated_remaining_minutes": 1.5255070120751584}
37
+ {"step": 740, "epoch": 0.39892183288409705, "timestamp": 1786741839.8993032, "loss": 2.3941486358642576, "grad_norm": 0.6171875, "learning_rate": 0.001, "train/total_time_seconds": 242.03530088812113, "train/time_per_step_avg": 0.3362794047221541, "train/epoch_time_elapsed": 523.2870299592614, "train/estimated_remaining_minutes": 1.4173238340295382}
38
+ {"step": 760, "epoch": 0.40970350404312667, "timestamp": 1786741854.293357, "loss": 2.3835357666015624, "grad_norm": 0.76171875, "learning_rate": 0.001, "train/total_time_seconds": 248.87612667307258, "train/time_per_step_avg": 0.3370570707693696, "train/epoch_time_elapsed": 537.6810839511454, "train/estimated_remaining_minutes": 1.3098743509109083}
39
+ {"step": 780, "epoch": 0.42048517520215634, "timestamp": 1786741868.3867362, "loss": 2.3709957122802736, "grad_norm": 0.58203125, "learning_rate": 0.001, "train/total_time_seconds": 255.26622416824102, "train/time_per_step_avg": 0.33136224407702686, "train/epoch_time_elapsed": 551.7744631879032, "train/estimated_remaining_minutes": 1.1999694298507058}
40
+ {"step": 800, "epoch": 0.431266846361186, "timestamp": 1786741882.7001128, "loss": 2.352398490905762, "grad_norm": 0.59375, "learning_rate": 0.001, "train/total_time_seconds": 261.9505532756448, "train/time_per_step_avg": 0.33118757780641317, "train/epoch_time_elapsed": 566.0878396071494, "train/estimated_remaining_minutes": 1.09146063864852}
41
+ {"step": 820, "epoch": 0.4420485175202156, "timestamp": 1786741897.277792, "loss": 2.3425289154052735, "grad_norm": 0.73046875, "learning_rate": 0.001, "train/total_time_seconds": 268.6989283747971, "train/time_per_step_avg": 0.3333498936891556, "train/epoch_time_elapsed": 580.6655190736055, "train/estimated_remaining_minutes": 0.9830448599077943}
42
+ {"step": 840, "epoch": 0.4528301886792453, "timestamp": 1786741911.5888515, "loss": 2.3336166381835937, "grad_norm": 0.66796875, "learning_rate": 0.001, "train/total_time_seconds": 275.3527529165149, "train/time_per_step_avg": 0.33317452028393746, "train/epoch_time_elapsed": 594.9765782393515, "train/estimated_remaining_minutes": 0.8741357235444915}
43
+ {"step": 860, "epoch": 0.4636118598382749, "timestamp": 1786741926.1444983, "loss": 2.3204221725463867, "grad_norm": 0.765625, "learning_rate": 0.001, "train/total_time_seconds": 282.25747611373663, "train/time_per_step_avg": 0.33381349440664054, "train/epoch_time_elapsed": 609.5322258919477, "train/estimated_remaining_minutes": 0.7658148576729289}
44
+ {"step": 880, "epoch": 0.4743935309973046, "timestamp": 1786741940.0126817, "loss": 2.3104717254638674, "grad_norm": 0.69921875, "learning_rate": 0.001, "train/total_time_seconds": 288.5647038668394, "train/time_per_step_avg": 0.3329847969859838, "train/epoch_time_elapsed": 623.4004086256027, "train/estimated_remaining_minutes": 0.6558288724246351}
45
+ {"step": 900, "epoch": 0.48517520215633425, "timestamp": 1786741954.5971494, "loss": 2.300767707824707, "grad_norm": 0.73828125, "learning_rate": 0.001, "train/total_time_seconds": 295.3995475657284, "train/time_per_step_avg": 0.33448994290083645, "train/epoch_time_elapsed": 637.9848755300045, "train/estimated_remaining_minutes": 0.5470361991957934}
46
+ {"step": 920, "epoch": 0.49595687331536387, "timestamp": 1786741969.1266496, "loss": 2.289361572265625, "grad_norm": 0.71875, "learning_rate": 0.001, "train/total_time_seconds": 302.08613530918956, "train/time_per_step_avg": 0.33387206934392455, "train/epoch_time_elapsed": 652.5143758803606, "train/estimated_remaining_minutes": 0.4378059932017239}
47
+ {"step": 940, "epoch": 0.5067385444743935, "timestamp": 1786741983.8404105, "loss": 2.283263397216797, "grad_norm": 0.58203125, "learning_rate": 0.001, "train/total_time_seconds": 309.0601658858359, "train/time_per_step_avg": 0.3370741296932101, "train/epoch_time_elapsed": 667.2281368896365, "train/estimated_remaining_minutes": 0.3287874105168467}
48
+ {"step": 960, "epoch": 0.5175202156334232, "timestamp": 1786741998.1476173, "loss": 2.28558235168457, "grad_norm": 0.546875, "learning_rate": 0.001, "train/total_time_seconds": 315.63702804222703, "train/time_per_step_avg": 0.33379551928490403, "train/epoch_time_elapsed": 681.5353437177837, "train/estimated_remaining_minutes": 0.21919238058487991}
49
+ {"step": 980, "epoch": 0.5283018867924528, "timestamp": 1786742012.7363431, "loss": 2.2729494094848635, "grad_norm": 0.60546875, "learning_rate": 0.001, "train/total_time_seconds": 322.51016880199313, "train/time_per_step_avg": 0.33945464935153724, "train/epoch_time_elapsed": 696.124069686979, "train/estimated_remaining_minutes": 0.10969733632720854}
50
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786742027.3704503, "loss": 2.262714385986328, "grad_norm": 0.78515625, "learning_rate": 0.001, "train/total_time_seconds": 329.34446839243174, "train/time_per_step_avg": 0.3394492082670331, "train/epoch_time_elapsed": 710.7581771910191, "train/estimated_remaining_minutes": 0.0}
51
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786742036.8943045, "eval_loss": 2.262706995010376, "eval_runtime": 9.5223, "eval_samples_per_second": 1000.494, "eval_steps_per_second": 1.995, "train/total_time_seconds": 329.34446839243174, "train/time_per_step_avg": 0.3394492082670331, "train/epoch_time_elapsed": 720.2820302955806, "train/estimated_remaining_minutes": 0.0}
52
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786742036.9414935, "train_runtime": 721.0312, "train_samples_per_second": 710.094, "train_steps_per_second": 1.387, "total_flos": 2323107545088000.0, "train_loss": 3.0280685653686525, "train/total_time_seconds": 329.34446839243174, "train/time_per_step_avg": 0.3394492082670331, "train/epoch_time_elapsed": 720.3292182870209, "train/estimated_remaining_minutes": 0.0}
53
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786742046.5003154, "eval_loss": 2.262706995010376, "eval_runtime": 9.5563, "eval_samples_per_second": 996.931, "eval_steps_per_second": 1.988, "train/total_time_seconds": 329.34446839243174, "train/time_per_step_avg": 0.3394492082670331, "train/epoch_time_elapsed": 729.8880413994193, "train/estimated_remaining_minutes": 0.0}