Hugging Face's logo

w-ahmad
/
art

Safetensors
Model card Files
xet
 Community
w-ahmad commited on
Commit
a906f50
·
verified ·
1 Parent(s): 475276d

Auto upload 2026-08-07T21:15:24.939620

Browse files
Files changed (36) hide show
  1. out/mlp-relu-21L_run/checkpoint-100/config.json +35 -0
  2. out/mlp-relu-21L_run/checkpoint-100/model.safetensors +3 -0
  3. out/mlp-relu-21L_run/checkpoint-100/optimizer.pt +3 -0
  4. out/mlp-relu-21L_run/checkpoint-100/rng_state.pth +3 -0
  5. out/mlp-relu-21L_run/checkpoint-100/scheduler.pt +3 -0
  6. out/mlp-relu-21L_run/checkpoint-100/tokenizer.json +0 -0
  7. out/mlp-relu-21L_run/checkpoint-100/tokenizer_config.json +13 -0
  8. out/mlp-relu-21L_run/checkpoint-100/trainer_state.json +85 -0
  9. out/mlp-relu-21L_run/checkpoint-100/training_args.bin +3 -0
  10. out/mlp-relu-21L_run/checkpoint-200/config.json +35 -0
  11. out/mlp-relu-21L_run/checkpoint-200/model.safetensors +3 -0
  12. out/mlp-relu-21L_run/checkpoint-200/optimizer.pt +3 -0
  13. out/mlp-relu-21L_run/checkpoint-200/rng_state.pth +3 -0
  14. out/mlp-relu-21L_run/checkpoint-200/scheduler.pt +3 -0
  15. out/mlp-relu-21L_run/checkpoint-200/tokenizer.json +0 -0
  16. out/mlp-relu-21L_run/checkpoint-200/tokenizer_config.json +13 -0
  17. out/mlp-relu-21L_run/checkpoint-200/trainer_state.json +136 -0
  18. out/mlp-relu-21L_run/checkpoint-200/training_args.bin +3 -0
  19. out/mlp-relu-21L_run/config.json +35 -0
  20. out/mlp-relu-21L_run/model.safetensors +3 -0
  21. out/mlp-relu-21L_run/training_args.bin +3 -0
  22. out/mlp-relu-21L_run/training_log.jsonl +0 -0
  23. out/mlp-silu-21L_run/README.md +75 -0
  24. out/mlp-silu-21L_run/checkpoint-1000/config.json +35 -0
  25. out/mlp-silu-21L_run/checkpoint-1000/model.safetensors +3 -0
  26. out/mlp-silu-21L_run/checkpoint-1000/optimizer.pt +3 -0
  27. out/mlp-silu-21L_run/checkpoint-1000/rng_state.pth +3 -0
  28. out/mlp-silu-21L_run/checkpoint-1000/scheduler.pt +3 -0
  29. out/mlp-silu-21L_run/checkpoint-1000/tokenizer.json +0 -0
  30. out/mlp-silu-21L_run/checkpoint-1000/tokenizer_config.json +13 -0
  31. out/mlp-silu-21L_run/checkpoint-1000/trainer_state.json +544 -0
  32. out/mlp-silu-21L_run/checkpoint-1000/training_args.bin +3 -0
  33. out/mlp-silu-21L_run/model.safetensors +1 -1
  34. out/mlp-silu-21L_run/tokenizer.json +0 -0
  35. out/mlp-silu-21L_run/tokenizer_config.json +13 -0
  36. out/mlp-silu-21L_run/training_log.jsonl +5 -0
out/mlp-relu-21L_run/checkpoint-100/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "relu",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 21,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-relu-21L_run/checkpoint-100/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c1d2168ba5ccd4dec55c70ceff20bb65a92badb78ef33e65e9ffc92e55e01a4
3
+ size 7959296
out/mlp-relu-21L_run/checkpoint-100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:acee0c9f5b2c0cd7182fc94bce4fddaa63ecff9c914797d28112e86e43d75721
3
+ size 16023738
out/mlp-relu-21L_run/checkpoint-100/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3cf9097d4513154245c48236b6ec5137b7ee2a21c9f58f2cba798ea275c6026f
3
+ size 14244
out/mlp-relu-21L_run/checkpoint-100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:906745ab5f61e73e8c1ba850ef3b839e2766eba889b49c843836266988895165
3
+ size 1064
out/mlp-relu-21L_run/checkpoint-100/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-relu-21L_run/checkpoint-100/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-relu-21L_run/checkpoint-100/trainer_state.json ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.05390835579514825,
6
+ "eval_steps": 50,
7
+ "global_step": 100,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9921875,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.159303283691406,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.34375,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.978606033325195,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.548793792724609,
29
+ "eval_runtime": 8.9393,
30
+ "eval_samples_per_second": 1065.738,
31
+ "eval_steps_per_second": 2.125,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.7734375,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.548751449584961,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 0.6953125,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.113357162475586,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 0.69140625,
51
+ "learning_rate": 0.001,
52
+ "loss": 4.717263793945312,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 4.535388946533203,
58
+ "eval_runtime": 8.9348,
59
+ "eval_samples_per_second": 1066.277,
60
+ "eval_steps_per_second": 2.127,
61
+ "step": 100
62
+ }
63
+ ],
64
+ "logging_steps": 20,
65
+ "max_steps": 1000,
66
+ "num_input_tokens_seen": 0,
67
+ "num_train_epochs": 1,
68
+ "save_steps": 100,
69
+ "stateful_callbacks": {
70
+ "TrainerControl": {
71
+ "args": {
72
+ "should_epoch_stop": false,
73
+ "should_evaluate": false,
74
+ "should_log": false,
75
+ "should_save": true,
76
+ "should_training_stop": false
77
+ },
78
+ "attributes": {}
79
+ }
80
+ },
81
+ "total_flos": 542031583641600.0,
82
+ "train_batch_size": 512,
83
+ "trial_name": null,
84
+ "trial_params": null
85
+ }
out/mlp-relu-21L_run/checkpoint-100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd6a52ebd8f733ff23c597a89bdde4b78f0bb3ad218ccc32dad91c595f91fb6f
3
+ size 4920
out/mlp-relu-21L_run/checkpoint-200/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "relu",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 21,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-relu-21L_run/checkpoint-200/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:231de5d78eb423f9116f5c4e8c66e5dc0d513a3c9ffe65951ace039cf60900cd
3
+ size 7959296
out/mlp-relu-21L_run/checkpoint-200/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e073001bbc9ec57d9559f3d1442bc77b2fae5a353bf40686b896362e3837c618
3
+ size 16023738
out/mlp-relu-21L_run/checkpoint-200/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecefbb3f17bb76b6655eb0157c98b5287c17fa4b4c72a6b9068b0823ce9fd18d
3
+ size 14244
out/mlp-relu-21L_run/checkpoint-200/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d3b5a970fc92af42664e84ba686059eaaacfaa668691c0cf6d2afddda677461
3
+ size 1064
out/mlp-relu-21L_run/checkpoint-200/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-relu-21L_run/checkpoint-200/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-relu-21L_run/checkpoint-200/trainer_state.json ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.1078167115902965,
6
+ "eval_steps": 50,
7
+ "global_step": 200,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9921875,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.159303283691406,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.34375,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.978606033325195,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.548793792724609,
29
+ "eval_runtime": 8.9393,
30
+ "eval_samples_per_second": 1065.738,
31
+ "eval_steps_per_second": 2.125,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.7734375,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.548751449584961,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 0.6953125,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.113357162475586,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 0.69140625,
51
+ "learning_rate": 0.001,
52
+ "loss": 4.717263793945312,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 4.535388946533203,
58
+ "eval_runtime": 8.9348,
59
+ "eval_samples_per_second": 1066.277,
60
+ "eval_steps_per_second": 2.127,
61
+ "step": 100
62
+ },
63
+ {
64
+ "epoch": 0.0646900269541779,
65
+ "grad_norm": 0.7734375,
66
+ "learning_rate": 0.001,
67
+ "loss": 4.405636596679687,
68
+ "step": 120
69
+ },
70
+ {
71
+ "epoch": 0.07547169811320754,
72
+ "grad_norm": 0.98046875,
73
+ "learning_rate": 0.001,
74
+ "loss": 4.175822448730469,
75
+ "step": 140
76
+ },
77
+ {
78
+ "epoch": 0.08086253369272237,
79
+ "eval_loss": 3.962047815322876,
80
+ "eval_runtime": 8.8885,
81
+ "eval_samples_per_second": 1071.834,
82
+ "eval_steps_per_second": 2.138,
83
+ "step": 150
84
+ },
85
+ {
86
+ "epoch": 0.0862533692722372,
87
+ "grad_norm": 0.70703125,
88
+ "learning_rate": 0.001,
89
+ "loss": 3.973344421386719,
90
+ "step": 160
91
+ },
92
+ {
93
+ "epoch": 0.09703504043126684,
94
+ "grad_norm": 0.66796875,
95
+ "learning_rate": 0.001,
96
+ "loss": 3.811610794067383,
97
+ "step": 180
98
+ },
99
+ {
100
+ "epoch": 0.1078167115902965,
101
+ "grad_norm": 0.8125,
102
+ "learning_rate": 0.001,
103
+ "loss": 3.6688804626464844,
104
+ "step": 200
105
+ },
106
+ {
107
+ "epoch": 0.1078167115902965,
108
+ "eval_loss": 3.61419939994812,
109
+ "eval_runtime": 9.0324,
110
+ "eval_samples_per_second": 1054.757,
111
+ "eval_steps_per_second": 2.104,
112
+ "step": 200
113
+ }
114
+ ],
115
+ "logging_steps": 20,
116
+ "max_steps": 1000,
117
+ "num_input_tokens_seen": 0,
118
+ "num_train_epochs": 1,
119
+ "save_steps": 100,
120
+ "stateful_callbacks": {
121
+ "TrainerControl": {
122
+ "args": {
123
+ "should_epoch_stop": false,
124
+ "should_evaluate": false,
125
+ "should_log": false,
126
+ "should_save": true,
127
+ "should_training_stop": false
128
+ },
129
+ "attributes": {}
130
+ }
131
+ },
132
+ "total_flos": 1084063167283200.0,
133
+ "train_batch_size": 512,
134
+ "trial_name": null,
135
+ "trial_params": null
136
+ }
out/mlp-relu-21L_run/checkpoint-200/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd6a52ebd8f733ff23c597a89bdde4b78f0bb3ad218ccc32dad91c595f91fb6f
3
+ size 4920
out/mlp-relu-21L_run/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "relu",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 21,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-relu-21L_run/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:231de5d78eb423f9116f5c4e8c66e5dc0d513a3c9ffe65951ace039cf60900cd
3
+ size 7959296
out/mlp-relu-21L_run/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd6a52ebd8f733ff23c597a89bdde4b78f0bb3ad218ccc32dad91c595f91fb6f
3
+ size 4920
out/mlp-relu-21L_run/training_log.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-silu-21L_run/README.md ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - generated_from_trainer
5
+ model-index:
6
+ - name: A-mlp-silu-21L
7
+ results: []
8
+ ---
9
+
10
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
11
+ should probably proofread and complete it, then remove this comment. -->
12
+
13
+ # A-mlp-silu-21L
14
+
15
+ This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
16
+ It achieves the following results on the evaluation set:
17
+ - Loss: 2.5113
18
+
19
+ ## Model description
20
+
21
+ More information needed
22
+
23
+ ## Intended uses & limitations
24
+
25
+ More information needed
26
+
27
+ ## Training and evaluation data
28
+
29
+ More information needed
30
+
31
+ ## Training procedure
32
+
33
+ ### Training hyperparameters
34
+
35
+ The following hyperparameters were used during training:
36
+ - learning_rate: 0.001
37
+ - train_batch_size: 512
38
+ - eval_batch_size: 512
39
+ - seed: 42
40
+ - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
41
+ - lr_scheduler_type: constant
42
+ - training_steps: 1000
43
+
44
+ ### Training results
45
+
46
+ | Training Loss | Epoch | Step | Validation Loss |
47
+ |:-------------:|:------:|:----:|:---------------:|
48
+ | 6.1069 | 0.0270 | 50 | 5.8308 |
49
+ | 5.4432 | 0.0539 | 100 | 5.3019 |
50
+ | 5.0078 | 0.0809 | 150 | 4.7696 |
51
+ | 4.3500 | 0.1078 | 200 | 4.2435 |
52
+ | 4.0161 | 0.1348 | 250 | 3.9000 |
53
+ | 3.6904 | 0.1617 | 300 | 3.6692 |
54
+ | 3.5370 | 0.1887 | 350 | 3.4723 |
55
+ | 3.3488 | 0.2156 | 400 | 3.3204 |
56
+ | 3.2442 | 0.2426 | 450 | 3.2016 |
57
+ | 3.1016 | 0.2695 | 500 | 3.0860 |
58
+ | 3.0244 | 0.2965 | 550 | 2.9901 |
59
+ | 2.9344 | 0.3235 | 600 | 2.9100 |
60
+ | 2.8818 | 0.3504 | 650 | 2.8379 |
61
+ | 2.7896 | 0.3774 | 700 | 2.7744 |
62
+ | 2.7577 | 0.4043 | 750 | 2.7147 |
63
+ | 2.6857 | 0.4313 | 800 | 2.6676 |
64
+ | 2.6519 | 0.4582 | 850 | 2.6167 |
65
+ | 2.5814 | 0.4852 | 900 | 2.5895 |
66
+ | 2.5651 | 0.5121 | 950 | 2.5468 |
67
+ | 2.5317 | 0.5391 | 1000 | 2.5113 |
68
+
69
+
70
+ ### Framework versions
71
+
72
+ - Transformers 5.15.0.dev0
73
+ - Pytorch 2.6.0+cu124
74
+ - Datasets 5.0.1
75
+ - Tokenizers 0.22.2
out/mlp-silu-21L_run/checkpoint-1000/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "silu",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 21,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-silu-21L_run/checkpoint-1000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2754009898894a1c1a143ccc88910789794c7f49be999e090fa79d94cadc4efb
3
+ size 7959296
out/mlp-silu-21L_run/checkpoint-1000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9f3f56c513801d6758c5f5f2c37b5562e33e23cff30078bcd5a010d9433cb88
3
+ size 16023738
out/mlp-silu-21L_run/checkpoint-1000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
3
+ size 14244
out/mlp-silu-21L_run/checkpoint-1000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fc3ca8af69cc9d003a05a139e7997960ab393fe31ae1d898b29038573704b4e
3
+ size 1064
out/mlp-silu-21L_run/checkpoint-1000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-silu-21L_run/checkpoint-1000/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-silu-21L_run/checkpoint-1000/trainer_state.json ADDED
@@ -0,0 +1,544 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.5390835579514824,
6
+ "eval_steps": 50,
7
+ "global_step": 1000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 1.0234375,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.275809478759766,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.7265625,
22
+ "learning_rate": 0.001,
23
+ "loss": 6.106889724731445,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.830765724182129,
29
+ "eval_runtime": 8.9498,
30
+ "eval_samples_per_second": 1064.493,
31
+ "eval_steps_per_second": 2.123,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.1982421875,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.841082763671875,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 1.40625,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.642723083496094,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 0.36328125,
51
+ "learning_rate": 0.001,
52
+ "loss": 5.443193054199218,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 5.301933765411377,
58
+ "eval_runtime": 8.8647,
59
+ "eval_samples_per_second": 1074.707,
60
+ "eval_steps_per_second": 2.143,
61
+ "step": 100
62
+ },
63
+ {
64
+ "epoch": 0.0646900269541779,
65
+ "grad_norm": 0.6171875,
66
+ "learning_rate": 0.001,
67
+ "loss": 5.226654052734375,
68
+ "step": 120
69
+ },
70
+ {
71
+ "epoch": 0.07547169811320754,
72
+ "grad_norm": 0.875,
73
+ "learning_rate": 0.001,
74
+ "loss": 5.007802963256836,
75
+ "step": 140
76
+ },
77
+ {
78
+ "epoch": 0.08086253369272237,
79
+ "eval_loss": 4.769636631011963,
80
+ "eval_runtime": 9.0201,
81
+ "eval_samples_per_second": 1056.199,
82
+ "eval_steps_per_second": 2.106,
83
+ "step": 150
84
+ },
85
+ {
86
+ "epoch": 0.0862533692722372,
87
+ "grad_norm": 1.046875,
88
+ "learning_rate": 0.001,
89
+ "loss": 4.7866966247558596,
90
+ "step": 160
91
+ },
92
+ {
93
+ "epoch": 0.09703504043126684,
94
+ "grad_norm": 0.42578125,
95
+ "learning_rate": 0.001,
96
+ "loss": 4.56334342956543,
97
+ "step": 180
98
+ },
99
+ {
100
+ "epoch": 0.1078167115902965,
101
+ "grad_norm": 0.59375,
102
+ "learning_rate": 0.001,
103
+ "loss": 4.34997444152832,
104
+ "step": 200
105
+ },
106
+ {
107
+ "epoch": 0.1078167115902965,
108
+ "eval_loss": 4.243469715118408,
109
+ "eval_runtime": 8.9202,
110
+ "eval_samples_per_second": 1068.025,
111
+ "eval_steps_per_second": 2.13,
112
+ "step": 200
113
+ },
114
+ {
115
+ "epoch": 0.11859838274932614,
116
+ "grad_norm": 0.46875,
117
+ "learning_rate": 0.001,
118
+ "loss": 4.168143844604492,
119
+ "step": 220
120
+ },
121
+ {
122
+ "epoch": 0.1293800539083558,
123
+ "grad_norm": 1.7421875,
124
+ "learning_rate": 0.001,
125
+ "loss": 4.016083145141602,
126
+ "step": 240
127
+ },
128
+ {
129
+ "epoch": 0.1347708894878706,
130
+ "eval_loss": 3.899991512298584,
131
+ "eval_runtime": 9.0184,
132
+ "eval_samples_per_second": 1056.398,
133
+ "eval_steps_per_second": 2.107,
134
+ "step": 250
135
+ },
136
+ {
137
+ "epoch": 0.14016172506738545,
138
+ "grad_norm": 0.431640625,
139
+ "learning_rate": 0.001,
140
+ "loss": 3.8943809509277343,
141
+ "step": 260
142
+ },
143
+ {
144
+ "epoch": 0.1509433962264151,
145
+ "grad_norm": 0.77734375,
146
+ "learning_rate": 0.001,
147
+ "loss": 3.7810901641845702,
148
+ "step": 280
149
+ },
150
+ {
151
+ "epoch": 0.16172506738544473,
152
+ "grad_norm": 0.8203125,
153
+ "learning_rate": 0.001,
154
+ "loss": 3.6904102325439454,
155
+ "step": 300
156
+ },
157
+ {
158
+ "epoch": 0.16172506738544473,
159
+ "eval_loss": 3.6692276000976562,
160
+ "eval_runtime": 8.9092,
161
+ "eval_samples_per_second": 1069.343,
162
+ "eval_steps_per_second": 2.133,
163
+ "step": 300
164
+ },
165
+ {
166
+ "epoch": 0.1725067385444744,
167
+ "grad_norm": 0.384765625,
168
+ "learning_rate": 0.001,
169
+ "loss": 3.6146160125732423,
170
+ "step": 320
171
+ },
172
+ {
173
+ "epoch": 0.18328840970350405,
174
+ "grad_norm": 0.466796875,
175
+ "learning_rate": 0.001,
176
+ "loss": 3.536983871459961,
177
+ "step": 340
178
+ },
179
+ {
180
+ "epoch": 0.18867924528301888,
181
+ "eval_loss": 3.4722838401794434,
182
+ "eval_runtime": 8.9789,
183
+ "eval_samples_per_second": 1061.038,
184
+ "eval_steps_per_second": 2.116,
185
+ "step": 350
186
+ },
187
+ {
188
+ "epoch": 0.1940700808625337,
189
+ "grad_norm": 0.51171875,
190
+ "learning_rate": 0.001,
191
+ "loss": 3.4680179595947265,
192
+ "step": 360
193
+ },
194
+ {
195
+ "epoch": 0.20485175202156333,
196
+ "grad_norm": 0.765625,
197
+ "learning_rate": 0.001,
198
+ "loss": 3.391448974609375,
199
+ "step": 380
200
+ },
201
+ {
202
+ "epoch": 0.215633423180593,
203
+ "grad_norm": 0.734375,
204
+ "learning_rate": 0.001,
205
+ "loss": 3.3488487243652343,
206
+ "step": 400
207
+ },
208
+ {
209
+ "epoch": 0.215633423180593,
210
+ "eval_loss": 3.320380687713623,
211
+ "eval_runtime": 8.9977,
212
+ "eval_samples_per_second": 1058.824,
213
+ "eval_steps_per_second": 2.112,
214
+ "step": 400
215
+ },
216
+ {
217
+ "epoch": 0.22641509433962265,
218
+ "grad_norm": 0.69140625,
219
+ "learning_rate": 0.001,
220
+ "loss": 3.280622863769531,
221
+ "step": 420
222
+ },
223
+ {
224
+ "epoch": 0.2371967654986523,
225
+ "grad_norm": 0.56640625,
226
+ "learning_rate": 0.001,
227
+ "loss": 3.2441749572753906,
228
+ "step": 440
229
+ },
230
+ {
231
+ "epoch": 0.24258760107816713,
232
+ "eval_loss": 3.2016375064849854,
233
+ "eval_runtime": 8.9622,
234
+ "eval_samples_per_second": 1063.019,
235
+ "eval_steps_per_second": 2.12,
236
+ "step": 450
237
+ },
238
+ {
239
+ "epoch": 0.24797843665768193,
240
+ "grad_norm": 0.43359375,
241
+ "learning_rate": 0.001,
242
+ "loss": 3.197739028930664,
243
+ "step": 460
244
+ },
245
+ {
246
+ "epoch": 0.2587601078167116,
247
+ "grad_norm": 1.359375,
248
+ "learning_rate": 0.001,
249
+ "loss": 3.14215202331543,
250
+ "step": 480
251
+ },
252
+ {
253
+ "epoch": 0.2695417789757412,
254
+ "grad_norm": 0.58203125,
255
+ "learning_rate": 0.001,
256
+ "loss": 3.1015945434570313,
257
+ "step": 500
258
+ },
259
+ {
260
+ "epoch": 0.2695417789757412,
261
+ "eval_loss": 3.0859992504119873,
262
+ "eval_runtime": 8.9249,
263
+ "eval_samples_per_second": 1067.459,
264
+ "eval_steps_per_second": 2.129,
265
+ "step": 500
266
+ },
267
+ {
268
+ "epoch": 0.2803234501347709,
269
+ "grad_norm": 0.55078125,
270
+ "learning_rate": 0.001,
271
+ "loss": 3.070932388305664,
272
+ "step": 520
273
+ },
274
+ {
275
+ "epoch": 0.29110512129380056,
276
+ "grad_norm": 0.625,
277
+ "learning_rate": 0.001,
278
+ "loss": 3.0243860244750977,
279
+ "step": 540
280
+ },
281
+ {
282
+ "epoch": 0.29649595687331537,
283
+ "eval_loss": 2.9901180267333984,
284
+ "eval_runtime": 8.9801,
285
+ "eval_samples_per_second": 1060.899,
286
+ "eval_steps_per_second": 2.116,
287
+ "step": 550
288
+ },
289
+ {
290
+ "epoch": 0.3018867924528302,
291
+ "grad_norm": 0.5859375,
292
+ "learning_rate": 0.001,
293
+ "loss": 2.991516876220703,
294
+ "step": 560
295
+ },
296
+ {
297
+ "epoch": 0.31266846361185985,
298
+ "grad_norm": 0.65234375,
299
+ "learning_rate": 0.001,
300
+ "loss": 2.9559335708618164,
301
+ "step": 580
302
+ },
303
+ {
304
+ "epoch": 0.32345013477088946,
305
+ "grad_norm": 0.474609375,
306
+ "learning_rate": 0.001,
307
+ "loss": 2.9344303131103517,
308
+ "step": 600
309
+ },
310
+ {
311
+ "epoch": 0.32345013477088946,
312
+ "eval_loss": 2.910011053085327,
313
+ "eval_runtime": 9.0222,
314
+ "eval_samples_per_second": 1055.947,
315
+ "eval_steps_per_second": 2.106,
316
+ "step": 600
317
+ },
318
+ {
319
+ "epoch": 0.33423180592991913,
320
+ "grad_norm": 0.486328125,
321
+ "learning_rate": 0.001,
322
+ "loss": 2.8876419067382812,
323
+ "step": 620
324
+ },
325
+ {
326
+ "epoch": 0.3450134770889488,
327
+ "grad_norm": 0.73828125,
328
+ "learning_rate": 0.001,
329
+ "loss": 2.8818475723266603,
330
+ "step": 640
331
+ },
332
+ {
333
+ "epoch": 0.3504043126684636,
334
+ "eval_loss": 2.837932586669922,
335
+ "eval_runtime": 9.3062,
336
+ "eval_samples_per_second": 1023.728,
337
+ "eval_steps_per_second": 2.042,
338
+ "step": 650
339
+ },
340
+ {
341
+ "epoch": 0.3557951482479784,
342
+ "grad_norm": 0.474609375,
343
+ "learning_rate": 0.001,
344
+ "loss": 2.8380828857421876,
345
+ "step": 660
346
+ },
347
+ {
348
+ "epoch": 0.3665768194070081,
349
+ "grad_norm": 0.6953125,
350
+ "learning_rate": 0.001,
351
+ "loss": 2.824846649169922,
352
+ "step": 680
353
+ },
354
+ {
355
+ "epoch": 0.37735849056603776,
356
+ "grad_norm": 0.6015625,
357
+ "learning_rate": 0.001,
358
+ "loss": 2.789586067199707,
359
+ "step": 700
360
+ },
361
+ {
362
+ "epoch": 0.37735849056603776,
363
+ "eval_loss": 2.774441957473755,
364
+ "eval_runtime": 9.0902,
365
+ "eval_samples_per_second": 1048.056,
366
+ "eval_steps_per_second": 2.09,
367
+ "step": 700
368
+ },
369
+ {
370
+ "epoch": 0.3881401617250674,
371
+ "grad_norm": 0.84765625,
372
+ "learning_rate": 0.001,
373
+ "loss": 2.7604854583740233,
374
+ "step": 720
375
+ },
376
+ {
377
+ "epoch": 0.39892183288409705,
378
+ "grad_norm": 0.55078125,
379
+ "learning_rate": 0.001,
380
+ "loss": 2.7577070236206054,
381
+ "step": 740
382
+ },
383
+ {
384
+ "epoch": 0.40431266846361186,
385
+ "eval_loss": 2.7147128582000732,
386
+ "eval_runtime": 9.4252,
387
+ "eval_samples_per_second": 1010.806,
388
+ "eval_steps_per_second": 2.016,
389
+ "step": 750
390
+ },
391
+ {
392
+ "epoch": 0.40970350404312667,
393
+ "grad_norm": 0.6796875,
394
+ "learning_rate": 0.001,
395
+ "loss": 2.714483451843262,
396
+ "step": 760
397
+ },
398
+ {
399
+ "epoch": 0.42048517520215634,
400
+ "grad_norm": 0.84375,
401
+ "learning_rate": 0.001,
402
+ "loss": 2.7089752197265624,
403
+ "step": 780
404
+ },
405
+ {
406
+ "epoch": 0.431266846361186,
407
+ "grad_norm": 0.470703125,
408
+ "learning_rate": 0.001,
409
+ "loss": 2.6856620788574217,
410
+ "step": 800
411
+ },
412
+ {
413
+ "epoch": 0.431266846361186,
414
+ "eval_loss": 2.667609691619873,
415
+ "eval_runtime": 9.466,
416
+ "eval_samples_per_second": 1006.439,
417
+ "eval_steps_per_second": 2.007,
418
+ "step": 800
419
+ },
420
+ {
421
+ "epoch": 0.4420485175202156,
422
+ "grad_norm": 0.875,
423
+ "learning_rate": 0.001,
424
+ "loss": 2.654224395751953,
425
+ "step": 820
426
+ },
427
+ {
428
+ "epoch": 0.4528301886792453,
429
+ "grad_norm": 0.62109375,
430
+ "learning_rate": 0.001,
431
+ "loss": 2.6519191741943358,
432
+ "step": 840
433
+ },
434
+ {
435
+ "epoch": 0.4582210242587601,
436
+ "eval_loss": 2.6166632175445557,
437
+ "eval_runtime": 9.0988,
438
+ "eval_samples_per_second": 1047.061,
439
+ "eval_steps_per_second": 2.088,
440
+ "step": 850
441
+ },
442
+ {
443
+ "epoch": 0.4636118598382749,
444
+ "grad_norm": 0.90625,
445
+ "learning_rate": 0.001,
446
+ "loss": 2.6152400970458984,
447
+ "step": 860
448
+ },
449
+ {
450
+ "epoch": 0.4743935309973046,
451
+ "grad_norm": 0.75,
452
+ "learning_rate": 0.001,
453
+ "loss": 2.6111644744873046,
454
+ "step": 880
455
+ },
456
+ {
457
+ "epoch": 0.48517520215633425,
458
+ "grad_norm": 0.77734375,
459
+ "learning_rate": 0.001,
460
+ "loss": 2.581392288208008,
461
+ "step": 900
462
+ },
463
+ {
464
+ "epoch": 0.48517520215633425,
465
+ "eval_loss": 2.589477777481079,
466
+ "eval_runtime": 9.1069,
467
+ "eval_samples_per_second": 1046.127,
468
+ "eval_steps_per_second": 2.086,
469
+ "step": 900
470
+ },
471
+ {
472
+ "epoch": 0.49595687331536387,
473
+ "grad_norm": 0.515625,
474
+ "learning_rate": 0.001,
475
+ "loss": 2.564067077636719,
476
+ "step": 920
477
+ },
478
+ {
479
+ "epoch": 0.5067385444743935,
480
+ "grad_norm": 0.60546875,
481
+ "learning_rate": 0.001,
482
+ "loss": 2.565108871459961,
483
+ "step": 940
484
+ },
485
+ {
486
+ "epoch": 0.5121293800539084,
487
+ "eval_loss": 2.5467922687530518,
488
+ "eval_runtime": 9.5854,
489
+ "eval_samples_per_second": 993.903,
490
+ "eval_steps_per_second": 1.982,
491
+ "step": 950
492
+ },
493
+ {
494
+ "epoch": 0.5175202156334232,
495
+ "grad_norm": 0.53125,
496
+ "learning_rate": 0.001,
497
+ "loss": 2.5423885345458985,
498
+ "step": 960
499
+ },
500
+ {
501
+ "epoch": 0.5283018867924528,
502
+ "grad_norm": 1.0,
503
+ "learning_rate": 0.001,
504
+ "loss": 2.5349870681762696,
505
+ "step": 980
506
+ },
507
+ {
508
+ "epoch": 0.5390835579514824,
509
+ "grad_norm": 0.59375,
510
+ "learning_rate": 0.001,
511
+ "loss": 2.5316877365112305,
512
+ "step": 1000
513
+ },
514
+ {
515
+ "epoch": 0.5390835579514824,
516
+ "eval_loss": 2.5112662315368652,
517
+ "eval_runtime": 9.4561,
518
+ "eval_samples_per_second": 1007.502,
519
+ "eval_steps_per_second": 2.009,
520
+ "step": 1000
521
+ }
522
+ ],
523
+ "logging_steps": 20,
524
+ "max_steps": 1000,
525
+ "num_input_tokens_seen": 0,
526
+ "num_train_epochs": 1,
527
+ "save_steps": 100,
528
+ "stateful_callbacks": {
529
+ "TrainerControl": {
530
+ "args": {
531
+ "should_epoch_stop": false,
532
+ "should_evaluate": false,
533
+ "should_log": false,
534
+ "should_save": true,
535
+ "should_training_stop": true
536
+ },
537
+ "attributes": {}
538
+ }
539
+ },
540
+ "total_flos": 5420315836416000.0,
541
+ "train_batch_size": 512,
542
+ "trial_name": null,
543
+ "trial_params": null
544
+ }
out/mlp-silu-21L_run/checkpoint-1000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a96c85db2a3e21d71e612f343aa2eb313bdb605656a66ec17aec0591f88ea5d6
3
+ size 4920
out/mlp-silu-21L_run/model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e0e05f1a0124dfa15180e526b834c981f30b63e748520cf2705c9c9bf6c2798f
3
  size 7959296
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2754009898894a1c1a143ccc88910789794c7f49be999e090fa79d94cadc4efb
3
  size 7959296
out/mlp-silu-21L_run/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-silu-21L_run/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-silu-21L_run/training_log.jsonl CHANGED
@@ -65,3 +65,8 @@
65
  {"step": 940, "epoch": 0.5067385444743935, "timestamp": 1786136985.4491959, "loss": 2.565108871459961, "grad_norm": 0.60546875, "learning_rate": 0.001, "train/total_time_seconds": 304.6231957823038, "train/time_per_step_avg": 0.3232784678786993, "train/epoch_time_elapsed": 827.0022541210055, "train/estimated_remaining_minutes": 0.3240672295556424}
66
  {"step": 950, "epoch": 0.5121293800539084, "timestamp": 1786137002.2163105, "eval_loss": 2.5467922687530518, "eval_runtime": 9.5854, "eval_samples_per_second": 993.903, "eval_steps_per_second": 1.982, "train/total_time_seconds": 307.85674518346786, "train/time_per_step_avg": 0.3233109885454178, "train/epoch_time_elapsed": 843.769368685782, "train/estimated_remaining_minutes": 0.2700497764767262}
67
  {"step": 960, "epoch": 0.5175202156334232, "timestamp": 1786137009.4476008, "loss": 2.5423885345458985, "grad_norm": 0.53125, "learning_rate": 0.001, "train/total_time_seconds": 311.0933882035315, "train/time_per_step_avg": 0.3233920970931649, "train/epoch_time_elapsed": 851.000660635531, "train/estimated_remaining_minutes": 0.21603707514134132}
 
 
 
 
 
 
65
  {"step": 940, "epoch": 0.5067385444743935, "timestamp": 1786136985.4491959, "loss": 2.565108871459961, "grad_norm": 0.60546875, "learning_rate": 0.001, "train/total_time_seconds": 304.6231957823038, "train/time_per_step_avg": 0.3232784678786993, "train/epoch_time_elapsed": 827.0022541210055, "train/estimated_remaining_minutes": 0.3240672295556424}
66
  {"step": 950, "epoch": 0.5121293800539084, "timestamp": 1786137002.2163105, "eval_loss": 2.5467922687530518, "eval_runtime": 9.5854, "eval_samples_per_second": 993.903, "eval_steps_per_second": 1.982, "train/total_time_seconds": 307.85674518346786, "train/time_per_step_avg": 0.3233109885454178, "train/epoch_time_elapsed": 843.769368685782, "train/estimated_remaining_minutes": 0.2700497764767262}
67
  {"step": 960, "epoch": 0.5175202156334232, "timestamp": 1786137009.4476008, "loss": 2.5423885345458985, "grad_norm": 0.53125, "learning_rate": 0.001, "train/total_time_seconds": 311.0933882035315, "train/time_per_step_avg": 0.3233920970931649, "train/epoch_time_elapsed": 851.000660635531, "train/estimated_remaining_minutes": 0.21603707514134132}
68
+ {"step": 980, "epoch": 0.5283018867924528, "timestamp": 1786137023.9689724, "loss": 2.5349870681762696, "grad_norm": 1.0, "learning_rate": 0.001, "train/total_time_seconds": 317.56865622103214, "train/time_per_step_avg": 0.3235281714424491, "train/epoch_time_elapsed": 865.5220307298005, "train/estimated_remaining_minutes": 0.10801654973504494}
69
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786137038.5877662, "loss": 2.5316877365112305, "grad_norm": 0.59375, "learning_rate": 0.001, "train/total_time_seconds": 324.0545369423926, "train/time_per_step_avg": 0.3237485465034842, "train/epoch_time_elapsed": 880.1408260278404, "train/estimated_remaining_minutes": 0.0}
70
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786137048.0455768, "eval_loss": 2.5112662315368652, "eval_runtime": 9.4561, "eval_samples_per_second": 1007.502, "eval_steps_per_second": 2.009, "train/total_time_seconds": 324.0545369423926, "train/time_per_step_avg": 0.3237485465034842, "train/epoch_time_elapsed": 889.598634827882, "train/estimated_remaining_minutes": 0.0}
71
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786137048.1345935, "train_runtime": 890.3483, "train_samples_per_second": 575.056, "train_steps_per_second": 1.123, "total_flos": 5420315836416000.0, "train_loss": 3.51598348236084, "train/total_time_seconds": 324.0545369423926, "train/time_per_step_avg": 0.3237485465034842, "train/epoch_time_elapsed": 889.6876515597105, "train/estimated_remaining_minutes": 0.0}
72
+ {"step": 1000, "epoch": 0.5390835579514824, "timestamp": 1786137059.1649728, "eval_loss": 2.5112662315368652, "eval_runtime": 9.4237, "eval_samples_per_second": 1010.961, "eval_steps_per_second": 2.016, "train/total_time_seconds": 324.0545369423926, "train/time_per_step_avg": 0.3237485465034842, "train/epoch_time_elapsed": 900.7180290110409, "train/estimated_remaining_minutes": 0.0}