Hugging Face's logo

w-ahmad
/
STRESS-TEST2

Safetensors
Model card Files
xet
 Community
w-ahmad commited on
Commit
f0c29f4
·
verified ·
1 Parent(s): aca8702

Auto upload zain 2026-08-14T20:53:52.460056 (part 3)

Browse files
Files changed (33) hide show
  1. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/config.json +36 -0
  2. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/model.safetensors +3 -0
  3. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/optimizer.pt +3 -0
  4. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/rng_state.pth +3 -0
  5. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/scheduler.pt +3 -0
  6. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/tokenizer.json +0 -0
  7. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/tokenizer_config.json +13 -0
  8. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/trainer_state.json +392 -0
  9. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/training_args.bin +3 -0
  10. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/config.json +36 -0
  11. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/model.safetensors +3 -0
  12. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/optimizer.pt +3 -0
  13. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/rng_state.pth +3 -0
  14. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/scheduler.pt +3 -0
  15. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/tokenizer.json +0 -0
  16. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/tokenizer_config.json +13 -0
  17. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/trainer_state.json +314 -0
  18. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/training_args.bin +3 -0
  19. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/config.json +36 -0
  20. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/model.safetensors +3 -0
  21. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/optimizer.pt +3 -0
  22. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/rng_state.pth +3 -0
  23. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/scheduler.pt +3 -0
  24. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/tokenizer.json +0 -0
  25. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/tokenizer_config.json +13 -0
  26. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/trainer_state.json +349 -0
  27. zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/training_args.bin +3 -0
  28. zain/Activation/out/mlp-silu-waleed10-9L_run/config.json +36 -0
  29. zain/Activation/out/mlp-silu-waleed10-9L_run/model.safetensors +3 -0
  30. zain/Activation/out/mlp-silu-waleed10-9L_run/tokenizer.json +0 -0
  31. zain/Activation/out/mlp-silu-waleed10-9L_run/tokenizer_config.json +13 -0
  32. zain/Activation/out/mlp-silu-waleed10-9L_run/training_args.bin +3 -0
  33. zain/Activation/out/mlp-silu-waleed10-9L_run/training_log.jsonl +14 -0
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "silu-waleed10",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12fb432a1dc0b881d2c4493f215386767274aec812acb5166f508a89c204f1d5
3
+ size 4010544
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:350d0d83091cbde23b2de9fb9a16f3a61999d91df2dafa2d0a9455bfdc85000c
3
+ size 8067258
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b66e3cc7c452b707ddac5caf0aa17618afb9bc1a0333600a22c4afb353f3165
3
+ size 14244
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fc3ca8af69cc9d003a05a139e7997960ab393fe31ae1d898b29038573704b4e
3
+ size 1064
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/trainer_state.json ADDED
@@ -0,0 +1,392 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.5390835579514824,
6
+ "eval_steps": 60000,
7
+ "global_step": 1000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 1.0078125,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.157280731201172,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.640625,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.922195816040039,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.03234501347708895,
28
+ "grad_norm": 0.578125,
29
+ "learning_rate": 0.001,
30
+ "loss": 5.382796859741211,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.0431266846361186,
35
+ "grad_norm": 0.65625,
36
+ "learning_rate": 0.001,
37
+ "loss": 4.915298080444336,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.05390835579514825,
42
+ "grad_norm": 1.5703125,
43
+ "learning_rate": 0.001,
44
+ "loss": 4.54681167602539,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.0646900269541779,
49
+ "grad_norm": 0.71875,
50
+ "learning_rate": 0.001,
51
+ "loss": 4.306855392456055,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.07547169811320754,
56
+ "grad_norm": 1.265625,
57
+ "learning_rate": 0.001,
58
+ "loss": 4.098614883422852,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.0862533692722372,
63
+ "grad_norm": 0.69140625,
64
+ "learning_rate": 0.001,
65
+ "loss": 3.9238147735595703,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.09703504043126684,
70
+ "grad_norm": 0.5234375,
71
+ "learning_rate": 0.001,
72
+ "loss": 3.7873252868652343,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.1078167115902965,
77
+ "grad_norm": 0.51953125,
78
+ "learning_rate": 0.001,
79
+ "loss": 3.674274444580078,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.11859838274932614,
84
+ "grad_norm": 0.71875,
85
+ "learning_rate": 0.001,
86
+ "loss": 3.5745738983154296,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.1293800539083558,
91
+ "grad_norm": 0.470703125,
92
+ "learning_rate": 0.001,
93
+ "loss": 3.4967296600341795,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.14016172506738545,
98
+ "grad_norm": 0.515625,
99
+ "learning_rate": 0.001,
100
+ "loss": 3.417258071899414,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.1509433962264151,
105
+ "grad_norm": 1.203125,
106
+ "learning_rate": 0.001,
107
+ "loss": 3.3540878295898438,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.16172506738544473,
112
+ "grad_norm": 0.59375,
113
+ "learning_rate": 0.001,
114
+ "loss": 3.29357795715332,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.1725067385444744,
119
+ "grad_norm": 1.0546875,
120
+ "learning_rate": 0.001,
121
+ "loss": 3.240766143798828,
122
+ "step": 320
123
+ },
124
+ {
125
+ "epoch": 0.18328840970350405,
126
+ "grad_norm": 0.65234375,
127
+ "learning_rate": 0.001,
128
+ "loss": 3.1872772216796874,
129
+ "step": 340
130
+ },
131
+ {
132
+ "epoch": 0.1940700808625337,
133
+ "grad_norm": 0.953125,
134
+ "learning_rate": 0.001,
135
+ "loss": 3.126861000061035,
136
+ "step": 360
137
+ },
138
+ {
139
+ "epoch": 0.20485175202156333,
140
+ "grad_norm": 0.73828125,
141
+ "learning_rate": 0.001,
142
+ "loss": 3.085362434387207,
143
+ "step": 380
144
+ },
145
+ {
146
+ "epoch": 0.215633423180593,
147
+ "grad_norm": 0.76953125,
148
+ "learning_rate": 0.001,
149
+ "loss": 3.034626007080078,
150
+ "step": 400
151
+ },
152
+ {
153
+ "epoch": 0.22641509433962265,
154
+ "grad_norm": 1.0078125,
155
+ "learning_rate": 0.001,
156
+ "loss": 3.0044570922851563,
157
+ "step": 420
158
+ },
159
+ {
160
+ "epoch": 0.2371967654986523,
161
+ "grad_norm": 0.90234375,
162
+ "learning_rate": 0.001,
163
+ "loss": 2.970414924621582,
164
+ "step": 440
165
+ },
166
+ {
167
+ "epoch": 0.24797843665768193,
168
+ "grad_norm": 0.94921875,
169
+ "learning_rate": 0.001,
170
+ "loss": 2.943356513977051,
171
+ "step": 460
172
+ },
173
+ {
174
+ "epoch": 0.2587601078167116,
175
+ "grad_norm": 1.3359375,
176
+ "learning_rate": 0.001,
177
+ "loss": 2.903603744506836,
178
+ "step": 480
179
+ },
180
+ {
181
+ "epoch": 0.2695417789757412,
182
+ "grad_norm": 1.0078125,
183
+ "learning_rate": 0.001,
184
+ "loss": 2.8795808792114257,
185
+ "step": 500
186
+ },
187
+ {
188
+ "epoch": 0.2803234501347709,
189
+ "grad_norm": 0.796875,
190
+ "learning_rate": 0.001,
191
+ "loss": 2.8545494079589844,
192
+ "step": 520
193
+ },
194
+ {
195
+ "epoch": 0.29110512129380056,
196
+ "grad_norm": 0.8203125,
197
+ "learning_rate": 0.001,
198
+ "loss": 2.8319740295410156,
199
+ "step": 540
200
+ },
201
+ {
202
+ "epoch": 0.3018867924528302,
203
+ "grad_norm": 0.78515625,
204
+ "learning_rate": 0.001,
205
+ "loss": 2.8041534423828125,
206
+ "step": 560
207
+ },
208
+ {
209
+ "epoch": 0.31266846361185985,
210
+ "grad_norm": 0.94140625,
211
+ "learning_rate": 0.001,
212
+ "loss": 2.7842403411865235,
213
+ "step": 580
214
+ },
215
+ {
216
+ "epoch": 0.32345013477088946,
217
+ "grad_norm": 0.93359375,
218
+ "learning_rate": 0.001,
219
+ "loss": 2.758980941772461,
220
+ "step": 600
221
+ },
222
+ {
223
+ "epoch": 0.33423180592991913,
224
+ "grad_norm": 0.9296875,
225
+ "learning_rate": 0.001,
226
+ "loss": 2.737656021118164,
227
+ "step": 620
228
+ },
229
+ {
230
+ "epoch": 0.3450134770889488,
231
+ "grad_norm": 0.8828125,
232
+ "learning_rate": 0.001,
233
+ "loss": 2.7211132049560547,
234
+ "step": 640
235
+ },
236
+ {
237
+ "epoch": 0.3557951482479784,
238
+ "grad_norm": 1.015625,
239
+ "learning_rate": 0.001,
240
+ "loss": 2.699817657470703,
241
+ "step": 660
242
+ },
243
+ {
244
+ "epoch": 0.3665768194070081,
245
+ "grad_norm": 0.734375,
246
+ "learning_rate": 0.001,
247
+ "loss": 2.68052978515625,
248
+ "step": 680
249
+ },
250
+ {
251
+ "epoch": 0.37735849056603776,
252
+ "grad_norm": 0.9375,
253
+ "learning_rate": 0.001,
254
+ "loss": 2.6656814575195313,
255
+ "step": 700
256
+ },
257
+ {
258
+ "epoch": 0.3881401617250674,
259
+ "grad_norm": 1.0546875,
260
+ "learning_rate": 0.001,
261
+ "loss": 2.6368873596191404,
262
+ "step": 720
263
+ },
264
+ {
265
+ "epoch": 0.39892183288409705,
266
+ "grad_norm": 0.88671875,
267
+ "learning_rate": 0.001,
268
+ "loss": 2.634653854370117,
269
+ "step": 740
270
+ },
271
+ {
272
+ "epoch": 0.40970350404312667,
273
+ "grad_norm": 0.984375,
274
+ "learning_rate": 0.001,
275
+ "loss": 2.613153076171875,
276
+ "step": 760
277
+ },
278
+ {
279
+ "epoch": 0.42048517520215634,
280
+ "grad_norm": 1.046875,
281
+ "learning_rate": 0.001,
282
+ "loss": 2.6026376724243163,
283
+ "step": 780
284
+ },
285
+ {
286
+ "epoch": 0.431266846361186,
287
+ "grad_norm": 0.92578125,
288
+ "learning_rate": 0.001,
289
+ "loss": 2.581294059753418,
290
+ "step": 800
291
+ },
292
+ {
293
+ "epoch": 0.4420485175202156,
294
+ "grad_norm": 0.8359375,
295
+ "learning_rate": 0.001,
296
+ "loss": 2.5675100326538085,
297
+ "step": 820
298
+ },
299
+ {
300
+ "epoch": 0.4528301886792453,
301
+ "grad_norm": 0.90625,
302
+ "learning_rate": 0.001,
303
+ "loss": 2.5610565185546874,
304
+ "step": 840
305
+ },
306
+ {
307
+ "epoch": 0.4636118598382749,
308
+ "grad_norm": 1.0,
309
+ "learning_rate": 0.001,
310
+ "loss": 2.5413028717041017,
311
+ "step": 860
312
+ },
313
+ {
314
+ "epoch": 0.4743935309973046,
315
+ "grad_norm": 1.0390625,
316
+ "learning_rate": 0.001,
317
+ "loss": 2.5273571014404297,
318
+ "step": 880
319
+ },
320
+ {
321
+ "epoch": 0.48517520215633425,
322
+ "grad_norm": 1.03125,
323
+ "learning_rate": 0.001,
324
+ "loss": 2.5183788299560548,
325
+ "step": 900
326
+ },
327
+ {
328
+ "epoch": 0.49595687331536387,
329
+ "grad_norm": 1.0859375,
330
+ "learning_rate": 0.001,
331
+ "loss": 2.500646209716797,
332
+ "step": 920
333
+ },
334
+ {
335
+ "epoch": 0.5067385444743935,
336
+ "grad_norm": 1.046875,
337
+ "learning_rate": 0.001,
338
+ "loss": 2.4960922241210937,
339
+ "step": 940
340
+ },
341
+ {
342
+ "epoch": 0.5175202156334232,
343
+ "grad_norm": 0.97265625,
344
+ "learning_rate": 0.001,
345
+ "loss": 2.491361618041992,
346
+ "step": 960
347
+ },
348
+ {
349
+ "epoch": 0.5283018867924528,
350
+ "grad_norm": 0.86328125,
351
+ "learning_rate": 0.001,
352
+ "loss": 2.4792640686035154,
353
+ "step": 980
354
+ },
355
+ {
356
+ "epoch": 0.5390835579514824,
357
+ "grad_norm": 1.0703125,
358
+ "learning_rate": 0.001,
359
+ "loss": 2.468957710266113,
360
+ "step": 1000
361
+ },
362
+ {
363
+ "epoch": 0.5390835579514824,
364
+ "eval_loss": 2.4668548107147217,
365
+ "eval_runtime": 9.2657,
366
+ "eval_samples_per_second": 1028.197,
367
+ "eval_steps_per_second": 2.051,
368
+ "step": 1000
369
+ }
370
+ ],
371
+ "logging_steps": 20,
372
+ "max_steps": 1000,
373
+ "num_input_tokens_seen": 0,
374
+ "num_train_epochs": 1,
375
+ "save_steps": 100,
376
+ "stateful_callbacks": {
377
+ "TrainerControl": {
378
+ "args": {
379
+ "should_epoch_stop": false,
380
+ "should_evaluate": false,
381
+ "should_log": false,
382
+ "should_save": true,
383
+ "should_training_stop": true
384
+ },
385
+ "attributes": {}
386
+ }
387
+ },
388
+ "total_flos": 2323107545088000.0,
389
+ "train_batch_size": 512,
390
+ "trial_name": null,
391
+ "trial_params": null
392
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-1000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:118cd2a20aa0afc7733fc42b014cb5bd002d9e415422b75f4251b66b12ce524d
3
+ size 4920
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "silu-waleed10",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86cd62a53620e109941b4218083a73669a2d82a3ef2d7ddbb36e7dcc6727d46e
3
+ size 4010544
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9dc3711a52369c78eae1086d9935046bd5bd363fdebfe5666b076d2e9afb4004
3
+ size 8067258
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
3
+ size 14244
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7334026bbd250befa466dcbe6bec8bfe84c03024263e4dea55e40698f2caa92e
3
+ size 1064
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/trainer_state.json ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.431266846361186,
6
+ "eval_steps": 60000,
7
+ "global_step": 800,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 1.0078125,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.157280731201172,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.640625,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.922195816040039,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.03234501347708895,
28
+ "grad_norm": 0.578125,
29
+ "learning_rate": 0.001,
30
+ "loss": 5.382796859741211,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.0431266846361186,
35
+ "grad_norm": 0.65625,
36
+ "learning_rate": 0.001,
37
+ "loss": 4.915298080444336,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.05390835579514825,
42
+ "grad_norm": 1.5703125,
43
+ "learning_rate": 0.001,
44
+ "loss": 4.54681167602539,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.0646900269541779,
49
+ "grad_norm": 0.71875,
50
+ "learning_rate": 0.001,
51
+ "loss": 4.306855392456055,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.07547169811320754,
56
+ "grad_norm": 1.265625,
57
+ "learning_rate": 0.001,
58
+ "loss": 4.098614883422852,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.0862533692722372,
63
+ "grad_norm": 0.69140625,
64
+ "learning_rate": 0.001,
65
+ "loss": 3.9238147735595703,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.09703504043126684,
70
+ "grad_norm": 0.5234375,
71
+ "learning_rate": 0.001,
72
+ "loss": 3.7873252868652343,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.1078167115902965,
77
+ "grad_norm": 0.51953125,
78
+ "learning_rate": 0.001,
79
+ "loss": 3.674274444580078,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.11859838274932614,
84
+ "grad_norm": 0.71875,
85
+ "learning_rate": 0.001,
86
+ "loss": 3.5745738983154296,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.1293800539083558,
91
+ "grad_norm": 0.470703125,
92
+ "learning_rate": 0.001,
93
+ "loss": 3.4967296600341795,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.14016172506738545,
98
+ "grad_norm": 0.515625,
99
+ "learning_rate": 0.001,
100
+ "loss": 3.417258071899414,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.1509433962264151,
105
+ "grad_norm": 1.203125,
106
+ "learning_rate": 0.001,
107
+ "loss": 3.3540878295898438,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.16172506738544473,
112
+ "grad_norm": 0.59375,
113
+ "learning_rate": 0.001,
114
+ "loss": 3.29357795715332,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.1725067385444744,
119
+ "grad_norm": 1.0546875,
120
+ "learning_rate": 0.001,
121
+ "loss": 3.240766143798828,
122
+ "step": 320
123
+ },
124
+ {
125
+ "epoch": 0.18328840970350405,
126
+ "grad_norm": 0.65234375,
127
+ "learning_rate": 0.001,
128
+ "loss": 3.1872772216796874,
129
+ "step": 340
130
+ },
131
+ {
132
+ "epoch": 0.1940700808625337,
133
+ "grad_norm": 0.953125,
134
+ "learning_rate": 0.001,
135
+ "loss": 3.126861000061035,
136
+ "step": 360
137
+ },
138
+ {
139
+ "epoch": 0.20485175202156333,
140
+ "grad_norm": 0.73828125,
141
+ "learning_rate": 0.001,
142
+ "loss": 3.085362434387207,
143
+ "step": 380
144
+ },
145
+ {
146
+ "epoch": 0.215633423180593,
147
+ "grad_norm": 0.76953125,
148
+ "learning_rate": 0.001,
149
+ "loss": 3.034626007080078,
150
+ "step": 400
151
+ },
152
+ {
153
+ "epoch": 0.22641509433962265,
154
+ "grad_norm": 1.0078125,
155
+ "learning_rate": 0.001,
156
+ "loss": 3.0044570922851563,
157
+ "step": 420
158
+ },
159
+ {
160
+ "epoch": 0.2371967654986523,
161
+ "grad_norm": 0.90234375,
162
+ "learning_rate": 0.001,
163
+ "loss": 2.970414924621582,
164
+ "step": 440
165
+ },
166
+ {
167
+ "epoch": 0.24797843665768193,
168
+ "grad_norm": 0.94921875,
169
+ "learning_rate": 0.001,
170
+ "loss": 2.943356513977051,
171
+ "step": 460
172
+ },
173
+ {
174
+ "epoch": 0.2587601078167116,
175
+ "grad_norm": 1.3359375,
176
+ "learning_rate": 0.001,
177
+ "loss": 2.903603744506836,
178
+ "step": 480
179
+ },
180
+ {
181
+ "epoch": 0.2695417789757412,
182
+ "grad_norm": 1.0078125,
183
+ "learning_rate": 0.001,
184
+ "loss": 2.8795808792114257,
185
+ "step": 500
186
+ },
187
+ {
188
+ "epoch": 0.2803234501347709,
189
+ "grad_norm": 0.796875,
190
+ "learning_rate": 0.001,
191
+ "loss": 2.8545494079589844,
192
+ "step": 520
193
+ },
194
+ {
195
+ "epoch": 0.29110512129380056,
196
+ "grad_norm": 0.8203125,
197
+ "learning_rate": 0.001,
198
+ "loss": 2.8319740295410156,
199
+ "step": 540
200
+ },
201
+ {
202
+ "epoch": 0.3018867924528302,
203
+ "grad_norm": 0.78515625,
204
+ "learning_rate": 0.001,
205
+ "loss": 2.8041534423828125,
206
+ "step": 560
207
+ },
208
+ {
209
+ "epoch": 0.31266846361185985,
210
+ "grad_norm": 0.94140625,
211
+ "learning_rate": 0.001,
212
+ "loss": 2.7842403411865235,
213
+ "step": 580
214
+ },
215
+ {
216
+ "epoch": 0.32345013477088946,
217
+ "grad_norm": 0.93359375,
218
+ "learning_rate": 0.001,
219
+ "loss": 2.758980941772461,
220
+ "step": 600
221
+ },
222
+ {
223
+ "epoch": 0.33423180592991913,
224
+ "grad_norm": 0.9296875,
225
+ "learning_rate": 0.001,
226
+ "loss": 2.737656021118164,
227
+ "step": 620
228
+ },
229
+ {
230
+ "epoch": 0.3450134770889488,
231
+ "grad_norm": 0.8828125,
232
+ "learning_rate": 0.001,
233
+ "loss": 2.7211132049560547,
234
+ "step": 640
235
+ },
236
+ {
237
+ "epoch": 0.3557951482479784,
238
+ "grad_norm": 1.015625,
239
+ "learning_rate": 0.001,
240
+ "loss": 2.699817657470703,
241
+ "step": 660
242
+ },
243
+ {
244
+ "epoch": 0.3665768194070081,
245
+ "grad_norm": 0.734375,
246
+ "learning_rate": 0.001,
247
+ "loss": 2.68052978515625,
248
+ "step": 680
249
+ },
250
+ {
251
+ "epoch": 0.37735849056603776,
252
+ "grad_norm": 0.9375,
253
+ "learning_rate": 0.001,
254
+ "loss": 2.6656814575195313,
255
+ "step": 700
256
+ },
257
+ {
258
+ "epoch": 0.3881401617250674,
259
+ "grad_norm": 1.0546875,
260
+ "learning_rate": 0.001,
261
+ "loss": 2.6368873596191404,
262
+ "step": 720
263
+ },
264
+ {
265
+ "epoch": 0.39892183288409705,
266
+ "grad_norm": 0.88671875,
267
+ "learning_rate": 0.001,
268
+ "loss": 2.634653854370117,
269
+ "step": 740
270
+ },
271
+ {
272
+ "epoch": 0.40970350404312667,
273
+ "grad_norm": 0.984375,
274
+ "learning_rate": 0.001,
275
+ "loss": 2.613153076171875,
276
+ "step": 760
277
+ },
278
+ {
279
+ "epoch": 0.42048517520215634,
280
+ "grad_norm": 1.046875,
281
+ "learning_rate": 0.001,
282
+ "loss": 2.6026376724243163,
283
+ "step": 780
284
+ },
285
+ {
286
+ "epoch": 0.431266846361186,
287
+ "grad_norm": 0.92578125,
288
+ "learning_rate": 0.001,
289
+ "loss": 2.581294059753418,
290
+ "step": 800
291
+ }
292
+ ],
293
+ "logging_steps": 20,
294
+ "max_steps": 1000,
295
+ "num_input_tokens_seen": 0,
296
+ "num_train_epochs": 1,
297
+ "save_steps": 100,
298
+ "stateful_callbacks": {
299
+ "TrainerControl": {
300
+ "args": {
301
+ "should_epoch_stop": false,
302
+ "should_evaluate": false,
303
+ "should_log": false,
304
+ "should_save": true,
305
+ "should_training_stop": false
306
+ },
307
+ "attributes": {}
308
+ }
309
+ },
310
+ "total_flos": 1858486036070400.0,
311
+ "train_batch_size": 512,
312
+ "trial_name": null,
313
+ "trial_params": null
314
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-800/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:118cd2a20aa0afc7733fc42b014cb5bd002d9e415422b75f4251b66b12ce524d
3
+ size 4920
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "silu-waleed10",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b0ca32f67afa733878429356708aee3bb5d38d80b847fb9641505a82b355fe8
3
+ size 4010544
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04c4a7bb9e716657d4b34d5dfae049d66c218ba4d4813c95c306adb2bf05d1fc
3
+ size 8067258
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
3
+ size 14244
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d48b00fea613f19335b87fb622648d28d7f8900963d07ba43c75df2b4dc3d4e
3
+ size 1064
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/trainer_state.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.48517520215633425,
6
+ "eval_steps": 60000,
7
+ "global_step": 900,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 1.0078125,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.157280731201172,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.640625,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.922195816040039,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.03234501347708895,
28
+ "grad_norm": 0.578125,
29
+ "learning_rate": 0.001,
30
+ "loss": 5.382796859741211,
31
+ "step": 60
32
+ },
33
+ {
34
+ "epoch": 0.0431266846361186,
35
+ "grad_norm": 0.65625,
36
+ "learning_rate": 0.001,
37
+ "loss": 4.915298080444336,
38
+ "step": 80
39
+ },
40
+ {
41
+ "epoch": 0.05390835579514825,
42
+ "grad_norm": 1.5703125,
43
+ "learning_rate": 0.001,
44
+ "loss": 4.54681167602539,
45
+ "step": 100
46
+ },
47
+ {
48
+ "epoch": 0.0646900269541779,
49
+ "grad_norm": 0.71875,
50
+ "learning_rate": 0.001,
51
+ "loss": 4.306855392456055,
52
+ "step": 120
53
+ },
54
+ {
55
+ "epoch": 0.07547169811320754,
56
+ "grad_norm": 1.265625,
57
+ "learning_rate": 0.001,
58
+ "loss": 4.098614883422852,
59
+ "step": 140
60
+ },
61
+ {
62
+ "epoch": 0.0862533692722372,
63
+ "grad_norm": 0.69140625,
64
+ "learning_rate": 0.001,
65
+ "loss": 3.9238147735595703,
66
+ "step": 160
67
+ },
68
+ {
69
+ "epoch": 0.09703504043126684,
70
+ "grad_norm": 0.5234375,
71
+ "learning_rate": 0.001,
72
+ "loss": 3.7873252868652343,
73
+ "step": 180
74
+ },
75
+ {
76
+ "epoch": 0.1078167115902965,
77
+ "grad_norm": 0.51953125,
78
+ "learning_rate": 0.001,
79
+ "loss": 3.674274444580078,
80
+ "step": 200
81
+ },
82
+ {
83
+ "epoch": 0.11859838274932614,
84
+ "grad_norm": 0.71875,
85
+ "learning_rate": 0.001,
86
+ "loss": 3.5745738983154296,
87
+ "step": 220
88
+ },
89
+ {
90
+ "epoch": 0.1293800539083558,
91
+ "grad_norm": 0.470703125,
92
+ "learning_rate": 0.001,
93
+ "loss": 3.4967296600341795,
94
+ "step": 240
95
+ },
96
+ {
97
+ "epoch": 0.14016172506738545,
98
+ "grad_norm": 0.515625,
99
+ "learning_rate": 0.001,
100
+ "loss": 3.417258071899414,
101
+ "step": 260
102
+ },
103
+ {
104
+ "epoch": 0.1509433962264151,
105
+ "grad_norm": 1.203125,
106
+ "learning_rate": 0.001,
107
+ "loss": 3.3540878295898438,
108
+ "step": 280
109
+ },
110
+ {
111
+ "epoch": 0.16172506738544473,
112
+ "grad_norm": 0.59375,
113
+ "learning_rate": 0.001,
114
+ "loss": 3.29357795715332,
115
+ "step": 300
116
+ },
117
+ {
118
+ "epoch": 0.1725067385444744,
119
+ "grad_norm": 1.0546875,
120
+ "learning_rate": 0.001,
121
+ "loss": 3.240766143798828,
122
+ "step": 320
123
+ },
124
+ {
125
+ "epoch": 0.18328840970350405,
126
+ "grad_norm": 0.65234375,
127
+ "learning_rate": 0.001,
128
+ "loss": 3.1872772216796874,
129
+ "step": 340
130
+ },
131
+ {
132
+ "epoch": 0.1940700808625337,
133
+ "grad_norm": 0.953125,
134
+ "learning_rate": 0.001,
135
+ "loss": 3.126861000061035,
136
+ "step": 360
137
+ },
138
+ {
139
+ "epoch": 0.20485175202156333,
140
+ "grad_norm": 0.73828125,
141
+ "learning_rate": 0.001,
142
+ "loss": 3.085362434387207,
143
+ "step": 380
144
+ },
145
+ {
146
+ "epoch": 0.215633423180593,
147
+ "grad_norm": 0.76953125,
148
+ "learning_rate": 0.001,
149
+ "loss": 3.034626007080078,
150
+ "step": 400
151
+ },
152
+ {
153
+ "epoch": 0.22641509433962265,
154
+ "grad_norm": 1.0078125,
155
+ "learning_rate": 0.001,
156
+ "loss": 3.0044570922851563,
157
+ "step": 420
158
+ },
159
+ {
160
+ "epoch": 0.2371967654986523,
161
+ "grad_norm": 0.90234375,
162
+ "learning_rate": 0.001,
163
+ "loss": 2.970414924621582,
164
+ "step": 440
165
+ },
166
+ {
167
+ "epoch": 0.24797843665768193,
168
+ "grad_norm": 0.94921875,
169
+ "learning_rate": 0.001,
170
+ "loss": 2.943356513977051,
171
+ "step": 460
172
+ },
173
+ {
174
+ "epoch": 0.2587601078167116,
175
+ "grad_norm": 1.3359375,
176
+ "learning_rate": 0.001,
177
+ "loss": 2.903603744506836,
178
+ "step": 480
179
+ },
180
+ {
181
+ "epoch": 0.2695417789757412,
182
+ "grad_norm": 1.0078125,
183
+ "learning_rate": 0.001,
184
+ "loss": 2.8795808792114257,
185
+ "step": 500
186
+ },
187
+ {
188
+ "epoch": 0.2803234501347709,
189
+ "grad_norm": 0.796875,
190
+ "learning_rate": 0.001,
191
+ "loss": 2.8545494079589844,
192
+ "step": 520
193
+ },
194
+ {
195
+ "epoch": 0.29110512129380056,
196
+ "grad_norm": 0.8203125,
197
+ "learning_rate": 0.001,
198
+ "loss": 2.8319740295410156,
199
+ "step": 540
200
+ },
201
+ {
202
+ "epoch": 0.3018867924528302,
203
+ "grad_norm": 0.78515625,
204
+ "learning_rate": 0.001,
205
+ "loss": 2.8041534423828125,
206
+ "step": 560
207
+ },
208
+ {
209
+ "epoch": 0.31266846361185985,
210
+ "grad_norm": 0.94140625,
211
+ "learning_rate": 0.001,
212
+ "loss": 2.7842403411865235,
213
+ "step": 580
214
+ },
215
+ {
216
+ "epoch": 0.32345013477088946,
217
+ "grad_norm": 0.93359375,
218
+ "learning_rate": 0.001,
219
+ "loss": 2.758980941772461,
220
+ "step": 600
221
+ },
222
+ {
223
+ "epoch": 0.33423180592991913,
224
+ "grad_norm": 0.9296875,
225
+ "learning_rate": 0.001,
226
+ "loss": 2.737656021118164,
227
+ "step": 620
228
+ },
229
+ {
230
+ "epoch": 0.3450134770889488,
231
+ "grad_norm": 0.8828125,
232
+ "learning_rate": 0.001,
233
+ "loss": 2.7211132049560547,
234
+ "step": 640
235
+ },
236
+ {
237
+ "epoch": 0.3557951482479784,
238
+ "grad_norm": 1.015625,
239
+ "learning_rate": 0.001,
240
+ "loss": 2.699817657470703,
241
+ "step": 660
242
+ },
243
+ {
244
+ "epoch": 0.3665768194070081,
245
+ "grad_norm": 0.734375,
246
+ "learning_rate": 0.001,
247
+ "loss": 2.68052978515625,
248
+ "step": 680
249
+ },
250
+ {
251
+ "epoch": 0.37735849056603776,
252
+ "grad_norm": 0.9375,
253
+ "learning_rate": 0.001,
254
+ "loss": 2.6656814575195313,
255
+ "step": 700
256
+ },
257
+ {
258
+ "epoch": 0.3881401617250674,
259
+ "grad_norm": 1.0546875,
260
+ "learning_rate": 0.001,
261
+ "loss": 2.6368873596191404,
262
+ "step": 720
263
+ },
264
+ {
265
+ "epoch": 0.39892183288409705,
266
+ "grad_norm": 0.88671875,
267
+ "learning_rate": 0.001,
268
+ "loss": 2.634653854370117,
269
+ "step": 740
270
+ },
271
+ {
272
+ "epoch": 0.40970350404312667,
273
+ "grad_norm": 0.984375,
274
+ "learning_rate": 0.001,
275
+ "loss": 2.613153076171875,
276
+ "step": 760
277
+ },
278
+ {
279
+ "epoch": 0.42048517520215634,
280
+ "grad_norm": 1.046875,
281
+ "learning_rate": 0.001,
282
+ "loss": 2.6026376724243163,
283
+ "step": 780
284
+ },
285
+ {
286
+ "epoch": 0.431266846361186,
287
+ "grad_norm": 0.92578125,
288
+ "learning_rate": 0.001,
289
+ "loss": 2.581294059753418,
290
+ "step": 800
291
+ },
292
+ {
293
+ "epoch": 0.4420485175202156,
294
+ "grad_norm": 0.8359375,
295
+ "learning_rate": 0.001,
296
+ "loss": 2.5675100326538085,
297
+ "step": 820
298
+ },
299
+ {
300
+ "epoch": 0.4528301886792453,
301
+ "grad_norm": 0.90625,
302
+ "learning_rate": 0.001,
303
+ "loss": 2.5610565185546874,
304
+ "step": 840
305
+ },
306
+ {
307
+ "epoch": 0.4636118598382749,
308
+ "grad_norm": 1.0,
309
+ "learning_rate": 0.001,
310
+ "loss": 2.5413028717041017,
311
+ "step": 860
312
+ },
313
+ {
314
+ "epoch": 0.4743935309973046,
315
+ "grad_norm": 1.0390625,
316
+ "learning_rate": 0.001,
317
+ "loss": 2.5273571014404297,
318
+ "step": 880
319
+ },
320
+ {
321
+ "epoch": 0.48517520215633425,
322
+ "grad_norm": 1.03125,
323
+ "learning_rate": 0.001,
324
+ "loss": 2.5183788299560548,
325
+ "step": 900
326
+ }
327
+ ],
328
+ "logging_steps": 20,
329
+ "max_steps": 1000,
330
+ "num_input_tokens_seen": 0,
331
+ "num_train_epochs": 1,
332
+ "save_steps": 100,
333
+ "stateful_callbacks": {
334
+ "TrainerControl": {
335
+ "args": {
336
+ "should_epoch_stop": false,
337
+ "should_evaluate": false,
338
+ "should_log": false,
339
+ "should_save": true,
340
+ "should_training_stop": false
341
+ },
342
+ "attributes": {}
343
+ }
344
+ },
345
+ "total_flos": 2090796790579200.0,
346
+ "train_batch_size": 512,
347
+ "trial_name": null,
348
+ "trial_params": null
349
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/checkpoint-900/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:118cd2a20aa0afc7733fc42b014cb5bd002d9e415422b75f4251b66b12ce524d
3
+ size 4920
zain/Activation/out/mlp-silu-waleed10-9L_run/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "silu-waleed10",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 9,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.16.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096,
35
+ "waleed_beta": 10.0
36
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12fb432a1dc0b881d2c4493f215386767274aec812acb5166f508a89c204f1d5
3
+ size 4010544
zain/Activation/out/mlp-silu-waleed10-9L_run/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
zain/Activation/out/mlp-silu-waleed10-9L_run/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
zain/Activation/out/mlp-silu-waleed10-9L_run/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:118cd2a20aa0afc7733fc42b014cb5bd002d9e415422b75f4251b66b12ce524d
3
+ size 4920
zain/Activation/out/mlp-silu-waleed10-9L_run/training_log.jsonl CHANGED
@@ -37,3 +37,17 @@
37
  {"step": 740, "epoch": 0.39892183288409705, "timestamp": 1786740495.4336789, "loss": 2.634653854370117, "grad_norm": 0.88671875, "learning_rate": 0.001, "train/total_time_seconds": 228.85845257341862, "train/time_per_step_avg": 0.3084182751178741, "train/epoch_time_elapsed": 509.6903050392866, "train/estimated_remaining_minutes": 1.340162109664163}
38
  {"step": 760, "epoch": 0.40970350404312667, "timestamp": 1786740509.337012, "loss": 2.613153076171875, "grad_norm": 0.984375, "learning_rate": 0.001, "train/total_time_seconds": 235.1386513710022, "train/time_per_step_avg": 0.30954349007457493, "train/epoch_time_elapsed": 523.5936387069523, "train/estimated_remaining_minutes": 1.2375718493210641}
39
  {"step": 780, "epoch": 0.42048517520215634, "timestamp": 1786740523.1417139, "loss": 2.6026376724243163, "grad_norm": 1.046875, "learning_rate": 0.001, "train/total_time_seconds": 241.27131348475814, "train/time_per_step_avg": 0.3093352550268173, "train/epoch_time_elapsed": 537.3983406051993, "train/estimated_remaining_minutes": 1.1341813881762135}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  {"step": 740, "epoch": 0.39892183288409705, "timestamp": 1786740495.4336789, "loss": 2.634653854370117, "grad_norm": 0.88671875, "learning_rate": 0.001, "train/total_time_seconds": 228.85845257341862, "train/time_per_step_avg": 0.3084182751178741, "train/epoch_time_elapsed": 509.6903050392866, "train/estimated_remaining_minutes": 1.340162109664163}
38
  {"step": 760, "epoch": 0.40970350404312667, "timestamp": 1786740509.337012, "loss": 2.613153076171875, "grad_norm": 0.984375, "learning_rate": 0.001, "train/total_time_seconds": 235.1386513710022, "train/time_per_step_avg": 0.30954349007457493, "train/epoch_time_elapsed": 523.5936387069523, "train/estimated_remaining_minutes": 1.2375718493210641}
39
  {"step": 780, "epoch": 0.42048517520215634, "timestamp": 1786740523.1417139, "loss": 2.6026376724243163, "grad_norm": 1.046875, "learning_rate": 0.001, "train/total_time_seconds": 241.27131348475814, "train/time_per_step_avg": 0.3093352550268173, "train/epoch_time_elapsed": 537.3983406051993, "train/estimated_remaining_minutes": 1.1341813881762135}
40
+ {"step": 800, "epoch": 0.431266846361186, "timestamp": 1786740536.937926, "loss": 2.581294059753418, "grad_norm": 0.92578125, "learning_rate": 0.001, "train/total_time_seconds": 247.4722731038928, "train/time_per_step_avg": 0.30901044368743896, "train/epoch_time_elapsed": 551.1945534646511, "train/estimated_remaining_minutes": 1.03113447126622}
41
+ {"step": 820, "epoch": 0.4420485175202156, "timestamp": 1786740550.7422278, "loss": 2.5675100326538085, "grad_norm": 0.8359375, "learning_rate": 0.001, "train/total_time_seconds": 253.5744924619794, "train/time_per_step_avg": 0.30868243768811227, "train/epoch_time_elapsed": 564.9988550879061, "train/estimated_remaining_minutes": 0.9277115577877294}
42
+ {"step": 840, "epoch": 0.4528301886792453, "timestamp": 1786740564.49888, "loss": 2.5610565185546874, "grad_norm": 0.90625, "learning_rate": 0.001, "train/total_time_seconds": 259.6634631752968, "train/time_per_step_avg": 0.3080501060187817, "train/epoch_time_elapsed": 578.7555079497397, "train/estimated_remaining_minutes": 0.8243284545247517}
43
+ {"step": 860, "epoch": 0.4636118598382749, "timestamp": 1786740578.3193977, "loss": 2.5413028717041017, "grad_norm": 1.0, "learning_rate": 0.001, "train/total_time_seconds": 265.90957019105554, "train/time_per_step_avg": 0.3077091882005334, "train/epoch_time_elapsed": 592.5760253742337, "train/estimated_remaining_minutes": 0.7214600741617786}
44
+ {"step": 880, "epoch": 0.4743935309973046, "timestamp": 1786740592.0811236, "loss": 2.5273571014404297, "grad_norm": 1.0390625, "learning_rate": 0.001, "train/total_time_seconds": 272.0700967460871, "train/time_per_step_avg": 0.3079878326132894, "train/epoch_time_elapsed": 606.3377512171865, "train/estimated_remaining_minutes": 0.6183411289683797}
45
+ {"step": 900, "epoch": 0.48517520215633425, "timestamp": 1786740605.8262074, "loss": 2.5183788299560548, "grad_norm": 1.03125, "learning_rate": 0.001, "train/total_time_seconds": 278.21320206671953, "train/time_per_step_avg": 0.3074092896282673, "train/epoch_time_elapsed": 620.0828354805708, "train/estimated_remaining_minutes": 0.5152096334568881}
46
+ {"step": 920, "epoch": 0.49595687331536387, "timestamp": 1786740619.7242038, "loss": 2.500646209716797, "grad_norm": 1.0859375, "learning_rate": 0.001, "train/total_time_seconds": 284.4278298318386, "train/time_per_step_avg": 0.3085333736985922, "train/epoch_time_elapsed": 633.9808309003711, "train/estimated_remaining_minutes": 0.4122142461330995}
47
+ {"step": 940, "epoch": 0.5067385444743935, "timestamp": 1786740633.4804492, "loss": 2.4960922241210937, "grad_norm": 1.046875, "learning_rate": 0.001, "train/total_time_seconds": 290.61977737396955, "train/time_per_step_avg": 0.30956314198672774, "train/epoch_time_elapsed": 647.7370772324502, "train/estimated_remaining_minutes": 0.30916997592975487}
48
+ {"step": 960, "epoch": 0.5175202156334232, "timestamp": 1786740647.0308967, "loss": 2.491361618041992, "grad_norm": 0.97265625, "learning_rate": 0.001, "train/total_time_seconds": 296.5974618829787, "train/time_per_step_avg": 0.3068789169192314, "train/epoch_time_elapsed": 661.2875243797898, "train/estimated_remaining_minutes": 0.20597045964095742}
49
+ {"step": 980, "epoch": 0.5283018867924528, "timestamp": 1786740661.116824, "loss": 2.4792640686035154, "grad_norm": 0.86328125, "learning_rate": 0.001, "train/total_time_seconds": 302.9764284156263, "train/time_per_step_avg": 0.3090633166953921, "train/epoch_time_elapsed": 675.3734518662095, "train/estimated_remaining_minutes": 0.10305320694409058}
50
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786740674.898659, "loss": 2.468957710266113, "grad_norm": 1.0703125, "learning_rate": 0.001, "train/total_time_seconds": 309.1270355395973, "train/time_per_step_avg": 0.3091383347287774, "train/epoch_time_elapsed": 689.1552871502936, "train/estimated_remaining_minutes": 0.0}
51
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786740684.1658895, "eval_loss": 2.4668548107147217, "eval_runtime": 9.2657, "eval_samples_per_second": 1028.197, "eval_steps_per_second": 2.051, "train/total_time_seconds": 309.1270355395973, "train/time_per_step_avg": 0.3091383347287774, "train/epoch_time_elapsed": 698.4225169718266, "train/estimated_remaining_minutes": 0.0}
52
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786740684.221405, "train_runtime": 699.2551, "train_samples_per_second": 732.208, "train_steps_per_second": 1.43, "total_flos": 2323107545088000.0, "train_loss": 3.2197410163879394, "train/total_time_seconds": 309.1270355395973, "train/time_per_step_avg": 0.3091383347287774, "train/epoch_time_elapsed": 698.4780311062932, "train/estimated_remaining_minutes": 0.0}
53
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786740693.774108, "eval_loss": 2.4668548107147217, "eval_runtime": 9.5503, "eval_samples_per_second": 997.557, "eval_steps_per_second": 1.989, "train/total_time_seconds": 309.1270355395973, "train/time_per_step_avg": 0.3091383347287774, "train/epoch_time_elapsed": 708.0307338684797, "train/estimated_remaining_minutes": 0.0}