Hugging Face's logo

w-ahmad
/
art

Safetensors
Model card Files
xet
 Community
w-ahmad commited on
Commit
0d1a6fe
·
verified ·
1 Parent(s): abf703b

Auto upload 2026-08-08T21:16:26.795996

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. __pycache__/exp.cpython-311.pyc +0 -0
  2. out/glu-linear-46L_run/checkpoint-100/config.json +35 -0
  3. out/glu-linear-46L_run/checkpoint-100/model.safetensors +3 -0
  4. out/glu-linear-46L_run/checkpoint-100/optimizer.pt +3 -0
  5. out/glu-linear-46L_run/checkpoint-100/rng_state.pth +3 -0
  6. out/glu-linear-46L_run/checkpoint-100/scheduler.pt +3 -0
  7. out/glu-linear-46L_run/checkpoint-100/tokenizer.json +0 -0
  8. out/glu-linear-46L_run/checkpoint-100/tokenizer_config.json +13 -0
  9. out/glu-linear-46L_run/checkpoint-100/trainer_state.json +85 -0
  10. out/glu-linear-46L_run/checkpoint-100/training_args.bin +3 -0
  11. out/glu-linear-46L_run/config.json +35 -0
  12. out/glu-linear-46L_run/model.safetensors +3 -0
  13. out/glu-linear-46L_run/training_args.bin +3 -0
  14. out/glu-linear-46L_run/training_log.jsonl +0 -0
  15. out/mlp-linear-46L_run/README.md +75 -0
  16. out/mlp-linear-46L_run/checkpoint-100/config.json +35 -0
  17. out/mlp-linear-46L_run/checkpoint-100/model.safetensors +3 -0
  18. out/mlp-linear-46L_run/checkpoint-100/optimizer.pt +3 -0
  19. out/mlp-linear-46L_run/checkpoint-100/rng_state.pth +3 -0
  20. out/mlp-linear-46L_run/checkpoint-100/scheduler.pt +3 -0
  21. out/mlp-linear-46L_run/checkpoint-100/tokenizer.json +0 -0
  22. out/mlp-linear-46L_run/checkpoint-100/tokenizer_config.json +13 -0
  23. out/mlp-linear-46L_run/checkpoint-100/trainer_state.json +85 -0
  24. out/mlp-linear-46L_run/checkpoint-100/training_args.bin +3 -0
  25. out/mlp-linear-46L_run/checkpoint-1000/config.json +35 -0
  26. out/mlp-linear-46L_run/checkpoint-1000/model.safetensors +3 -0
  27. out/mlp-linear-46L_run/checkpoint-1000/optimizer.pt +3 -0
  28. out/mlp-linear-46L_run/checkpoint-1000/rng_state.pth +3 -0
  29. out/mlp-linear-46L_run/checkpoint-1000/scheduler.pt +3 -0
  30. out/mlp-linear-46L_run/checkpoint-1000/tokenizer.json +0 -0
  31. out/mlp-linear-46L_run/checkpoint-1000/tokenizer_config.json +13 -0
  32. out/mlp-linear-46L_run/checkpoint-1000/trainer_state.json +544 -0
  33. out/mlp-linear-46L_run/checkpoint-1000/training_args.bin +3 -0
  34. out/mlp-linear-46L_run/checkpoint-200/config.json +35 -0
  35. out/mlp-linear-46L_run/checkpoint-200/model.safetensors +3 -0
  36. out/mlp-linear-46L_run/checkpoint-200/optimizer.pt +3 -0
  37. out/mlp-linear-46L_run/checkpoint-200/rng_state.pth +3 -0
  38. out/mlp-linear-46L_run/checkpoint-200/scheduler.pt +3 -0
  39. out/mlp-linear-46L_run/checkpoint-200/tokenizer.json +0 -0
  40. out/mlp-linear-46L_run/checkpoint-200/tokenizer_config.json +13 -0
  41. out/mlp-linear-46L_run/checkpoint-200/trainer_state.json +136 -0
  42. out/mlp-linear-46L_run/checkpoint-200/training_args.bin +3 -0
  43. out/mlp-linear-46L_run/checkpoint-300/config.json +35 -0
  44. out/mlp-linear-46L_run/checkpoint-300/model.safetensors +3 -0
  45. out/mlp-linear-46L_run/checkpoint-300/optimizer.pt +3 -0
  46. out/mlp-linear-46L_run/checkpoint-300/rng_state.pth +3 -0
  47. out/mlp-linear-46L_run/checkpoint-300/scheduler.pt +3 -0
  48. out/mlp-linear-46L_run/checkpoint-300/tokenizer.json +0 -0
  49. out/mlp-linear-46L_run/checkpoint-300/tokenizer_config.json +13 -0
  50. out/mlp-linear-46L_run/checkpoint-300/trainer_state.json +187 -0
__pycache__/exp.cpython-311.pyc CHANGED
Binary files a/__pycache__/exp.cpython-311.pyc and b/__pycache__/exp.cpython-311.pyc differ
 
out/glu-linear-46L_run/checkpoint-100/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "linear",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "glu",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 46,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/glu-linear-46L_run/checkpoint-100/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c2744d5e09f61ba614f87bff0144e288922370df35f92682c13f6b8567870e6
3
+ size 16191120
out/glu-linear-46L_run/checkpoint-100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81a25f72f669225bd137f7e8a1af66a8c51c91b2bef769be71361140092f3096
3
+ size 32644914
out/glu-linear-46L_run/checkpoint-100/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3cf9097d4513154245c48236b6ec5137b7ee2a21c9f58f2cba798ea275c6026f
3
+ size 14244
out/glu-linear-46L_run/checkpoint-100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:906745ab5f61e73e8c1ba850ef3b839e2766eba889b49c843836266988895165
3
+ size 1064
out/glu-linear-46L_run/checkpoint-100/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/glu-linear-46L_run/checkpoint-100/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/glu-linear-46L_run/checkpoint-100/trainer_state.json ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.05390835579514825,
6
+ "eval_steps": 50,
7
+ "global_step": 100,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.97265625,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.1971282958984375,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.185546875,
22
+ "learning_rate": 0.001,
23
+ "loss": 6.086066055297851,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.877857208251953,
29
+ "eval_runtime": 11.0222,
30
+ "eval_samples_per_second": 864.344,
31
+ "eval_steps_per_second": 1.724,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.22265625,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.883237075805664,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 0.404296875,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.661934661865234,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 1.609375,
51
+ "learning_rate": 0.001,
52
+ "loss": 5.3743633270263675,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 5.2120513916015625,
58
+ "eval_runtime": 10.9178,
59
+ "eval_samples_per_second": 872.615,
60
+ "eval_steps_per_second": 1.74,
61
+ "step": 100
62
+ }
63
+ ],
64
+ "logging_steps": 20,
65
+ "max_steps": 1000,
66
+ "num_input_tokens_seen": 0,
67
+ "num_train_epochs": 1,
68
+ "save_steps": 100,
69
+ "stateful_callbacks": {
70
+ "TrainerControl": {
71
+ "args": {
72
+ "should_epoch_stop": false,
73
+ "should_evaluate": false,
74
+ "should_log": false,
75
+ "should_save": true,
76
+ "should_training_stop": false
77
+ },
78
+ "attributes": {}
79
+ }
80
+ },
81
+ "total_flos": 1187283311001600.0,
82
+ "train_batch_size": 512,
83
+ "trial_name": null,
84
+ "trial_params": null
85
+ }
out/glu-linear-46L_run/checkpoint-100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40fd324a4072287822a976b80bc919dbd0f0d035235f9ed1eb047291bfce3b0d
3
+ size 4920
out/glu-linear-46L_run/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "linear",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "glu",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 46,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/glu-linear-46L_run/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c2744d5e09f61ba614f87bff0144e288922370df35f92682c13f6b8567870e6
3
+ size 16191120
out/glu-linear-46L_run/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40fd324a4072287822a976b80bc919dbd0f0d035235f9ed1eb047291bfce3b0d
3
+ size 4920
out/glu-linear-46L_run/training_log.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-linear-46L_run/README.md ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ tags:
4
+ - generated_from_trainer
5
+ model-index:
6
+ - name: A-mlp-linear-46L
7
+ results: []
8
+ ---
9
+
10
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
11
+ should probably proofread and complete it, then remove this comment. -->
12
+
13
+ # A-mlp-linear-46L
14
+
15
+ This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
16
+ It achieves the following results on the evaluation set:
17
+ - Loss: 2.8153
18
+
19
+ ## Model description
20
+
21
+ More information needed
22
+
23
+ ## Intended uses & limitations
24
+
25
+ More information needed
26
+
27
+ ## Training and evaluation data
28
+
29
+ More information needed
30
+
31
+ ## Training procedure
32
+
33
+ ### Training hyperparameters
34
+
35
+ The following hyperparameters were used during training:
36
+ - learning_rate: 0.001
37
+ - train_batch_size: 512
38
+ - eval_batch_size: 512
39
+ - seed: 42
40
+ - optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
41
+ - lr_scheduler_type: constant
42
+ - training_steps: 1000
43
+
44
+ ### Training results
45
+
46
+ | Training Loss | Epoch | Step | Validation Loss |
47
+ |:-------------:|:------:|:----:|:---------------:|
48
+ | 5.9943 | 0.0270 | 50 | 5.6660 |
49
+ | 5.4216 | 0.0539 | 100 | 5.3673 |
50
+ | 5.1264 | 0.0809 | 150 | 4.9933 |
51
+ | 4.7248 | 0.1078 | 200 | 4.6469 |
52
+ | 4.4794 | 0.1348 | 250 | 4.4149 |
53
+ | 4.1928 | 0.1617 | 300 | 4.1563 |
54
+ | 4.0202 | 0.1887 | 350 | 3.9751 |
55
+ | 3.7941 | 0.2156 | 400 | 3.7661 |
56
+ | 3.6807 | 0.2426 | 450 | 3.6146 |
57
+ | 3.5234 | 0.2695 | 500 | 3.4892 |
58
+ | 3.4312 | 0.2965 | 550 | 3.3768 |
59
+ | 3.3176 | 0.3235 | 600 | 3.3391 |
60
+ | 3.2724 | 0.3504 | 650 | 3.2303 |
61
+ | 3.1910 | 0.3774 | 700 | 3.1677 |
62
+ | 3.1384 | 0.4043 | 750 | 3.0951 |
63
+ | 3.0459 | 0.4313 | 800 | 3.0226 |
64
+ | 3.0106 | 0.4582 | 850 | 2.9586 |
65
+ | 2.9290 | 0.4852 | 900 | 2.9079 |
66
+ | 2.8933 | 0.5121 | 950 | 2.8532 |
67
+ | 2.8351 | 0.5391 | 1000 | 2.8153 |
68
+
69
+
70
+ ### Framework versions
71
+
72
+ - Transformers 5.15.0.dev0
73
+ - Pytorch 2.6.0+cu124
74
+ - Datasets 5.0.1
75
+ - Tokenizers 0.22.2
out/mlp-linear-46L_run/checkpoint-100/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "linear",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 46,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-linear-46L_run/checkpoint-100/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0275ef47dffd2ae2c2a3a7fed09769ef1ca9c40dba89d6184f59feae1fea65e
3
+ size 16186168
out/mlp-linear-46L_run/checkpoint-100/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f210fcf0f748cafe438c98c91637269bf9e3ab3135bfca5cc7182b248beddbaa
3
+ size 32605416
out/mlp-linear-46L_run/checkpoint-100/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3cf9097d4513154245c48236b6ec5137b7ee2a21c9f58f2cba798ea275c6026f
3
+ size 14244
out/mlp-linear-46L_run/checkpoint-100/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:906745ab5f61e73e8c1ba850ef3b839e2766eba889b49c843836266988895165
3
+ size 1064
out/mlp-linear-46L_run/checkpoint-100/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-linear-46L_run/checkpoint-100/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-linear-46L_run/checkpoint-100/trainer_state.json ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.05390835579514825,
6
+ "eval_steps": 50,
7
+ "global_step": 100,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9921875,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.2093955993652346,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.337890625,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.994318008422852,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.666038990020752,
29
+ "eval_runtime": 10.8497,
30
+ "eval_samples_per_second": 878.09,
31
+ "eval_steps_per_second": 1.751,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.6640625,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.679982757568359,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 0.4921875,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.49438705444336,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 1.3125,
51
+ "learning_rate": 0.001,
52
+ "loss": 5.421606826782226,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 5.367257118225098,
58
+ "eval_runtime": 10.9702,
59
+ "eval_samples_per_second": 868.441,
60
+ "eval_steps_per_second": 1.732,
61
+ "step": 100
62
+ }
63
+ ],
64
+ "logging_steps": 20,
65
+ "max_steps": 1000,
66
+ "num_input_tokens_seen": 0,
67
+ "num_train_epochs": 1,
68
+ "save_steps": 100,
69
+ "stateful_callbacks": {
70
+ "TrainerControl": {
71
+ "args": {
72
+ "should_epoch_stop": false,
73
+ "should_evaluate": false,
74
+ "should_log": false,
75
+ "should_save": true,
76
+ "should_training_stop": false
77
+ },
78
+ "attributes": {}
79
+ }
80
+ },
81
+ "total_flos": 1187283311001600.0,
82
+ "train_batch_size": 512,
83
+ "trial_name": null,
84
+ "trial_params": null
85
+ }
out/mlp-linear-46L_run/checkpoint-100/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a560522de5d1f6fee179c6909f9239eea63d45e591dc18243e116c36963af413
3
+ size 4920
out/mlp-linear-46L_run/checkpoint-1000/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "linear",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 46,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-linear-46L_run/checkpoint-1000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:755fec232932f05b4d82ff4fa051e3caf5f90b084b7deb3b8a91cd5c9b8e2cba
3
+ size 16186168
out/mlp-linear-46L_run/checkpoint-1000/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ceb7b621cc61684ab69f81cd5cc108ae90fce1ba8c846a58cfcc9a23fc1bc042
3
+ size 32605416
out/mlp-linear-46L_run/checkpoint-1000/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
3
+ size 14244
out/mlp-linear-46L_run/checkpoint-1000/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0fc3ca8af69cc9d003a05a139e7997960ab393fe31ae1d898b29038573704b4e
3
+ size 1064
out/mlp-linear-46L_run/checkpoint-1000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-linear-46L_run/checkpoint-1000/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-linear-46L_run/checkpoint-1000/trainer_state.json ADDED
@@ -0,0 +1,544 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.5390835579514824,
6
+ "eval_steps": 50,
7
+ "global_step": 1000,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9921875,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.2093955993652346,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.337890625,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.994318008422852,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.666038990020752,
29
+ "eval_runtime": 10.8497,
30
+ "eval_samples_per_second": 878.09,
31
+ "eval_steps_per_second": 1.751,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.6640625,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.679982757568359,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 0.4921875,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.49438705444336,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 1.3125,
51
+ "learning_rate": 0.001,
52
+ "loss": 5.421606826782226,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 5.367257118225098,
58
+ "eval_runtime": 10.9702,
59
+ "eval_samples_per_second": 868.441,
60
+ "eval_steps_per_second": 1.732,
61
+ "step": 100
62
+ },
63
+ {
64
+ "epoch": 0.0646900269541779,
65
+ "grad_norm": 1.1640625,
66
+ "learning_rate": 0.001,
67
+ "loss": 5.309521102905274,
68
+ "step": 120
69
+ },
70
+ {
71
+ "epoch": 0.07547169811320754,
72
+ "grad_norm": 1.4140625,
73
+ "learning_rate": 0.001,
74
+ "loss": 5.126371765136719,
75
+ "step": 140
76
+ },
77
+ {
78
+ "epoch": 0.08086253369272237,
79
+ "eval_loss": 4.993337154388428,
80
+ "eval_runtime": 10.9852,
81
+ "eval_samples_per_second": 867.26,
82
+ "eval_steps_per_second": 1.73,
83
+ "step": 150
84
+ },
85
+ {
86
+ "epoch": 0.0862533692722372,
87
+ "grad_norm": 1.5078125,
88
+ "learning_rate": 0.001,
89
+ "loss": 4.988518524169922,
90
+ "step": 160
91
+ },
92
+ {
93
+ "epoch": 0.09703504043126684,
94
+ "grad_norm": 1.8828125,
95
+ "learning_rate": 0.001,
96
+ "loss": 4.8510185241699215,
97
+ "step": 180
98
+ },
99
+ {
100
+ "epoch": 0.1078167115902965,
101
+ "grad_norm": 1.28125,
102
+ "learning_rate": 0.001,
103
+ "loss": 4.724799346923828,
104
+ "step": 200
105
+ },
106
+ {
107
+ "epoch": 0.1078167115902965,
108
+ "eval_loss": 4.646933555603027,
109
+ "eval_runtime": 10.9059,
110
+ "eval_samples_per_second": 873.563,
111
+ "eval_steps_per_second": 1.742,
112
+ "step": 200
113
+ },
114
+ {
115
+ "epoch": 0.11859838274932614,
116
+ "grad_norm": 3.03125,
117
+ "learning_rate": 0.001,
118
+ "loss": 4.576333618164062,
119
+ "step": 220
120
+ },
121
+ {
122
+ "epoch": 0.1293800539083558,
123
+ "grad_norm": 3.28125,
124
+ "learning_rate": 0.001,
125
+ "loss": 4.479412841796875,
126
+ "step": 240
127
+ },
128
+ {
129
+ "epoch": 0.1347708894878706,
130
+ "eval_loss": 4.414884567260742,
131
+ "eval_runtime": 11.1112,
132
+ "eval_samples_per_second": 857.424,
133
+ "eval_steps_per_second": 1.71,
134
+ "step": 250
135
+ },
136
+ {
137
+ "epoch": 0.14016172506738545,
138
+ "grad_norm": 1.703125,
139
+ "learning_rate": 0.001,
140
+ "loss": 4.398517608642578,
141
+ "step": 260
142
+ },
143
+ {
144
+ "epoch": 0.1509433962264151,
145
+ "grad_norm": 0.416015625,
146
+ "learning_rate": 0.001,
147
+ "loss": 4.271953582763672,
148
+ "step": 280
149
+ },
150
+ {
151
+ "epoch": 0.16172506738544473,
152
+ "grad_norm": 0.63671875,
153
+ "learning_rate": 0.001,
154
+ "loss": 4.192776870727539,
155
+ "step": 300
156
+ },
157
+ {
158
+ "epoch": 0.16172506738544473,
159
+ "eval_loss": 4.156349182128906,
160
+ "eval_runtime": 10.9589,
161
+ "eval_samples_per_second": 869.336,
162
+ "eval_steps_per_second": 1.734,
163
+ "step": 300
164
+ },
165
+ {
166
+ "epoch": 0.1725067385444744,
167
+ "grad_norm": 1.2734375,
168
+ "learning_rate": 0.001,
169
+ "loss": 4.114960479736328,
170
+ "step": 320
171
+ },
172
+ {
173
+ "epoch": 0.18328840970350405,
174
+ "grad_norm": 0.376953125,
175
+ "learning_rate": 0.001,
176
+ "loss": 4.020223999023438,
177
+ "step": 340
178
+ },
179
+ {
180
+ "epoch": 0.18867924528301888,
181
+ "eval_loss": 3.975097179412842,
182
+ "eval_runtime": 10.9734,
183
+ "eval_samples_per_second": 868.192,
184
+ "eval_steps_per_second": 1.731,
185
+ "step": 350
186
+ },
187
+ {
188
+ "epoch": 0.1940700808625337,
189
+ "grad_norm": 0.6328125,
190
+ "learning_rate": 0.001,
191
+ "loss": 3.9757205963134767,
192
+ "step": 360
193
+ },
194
+ {
195
+ "epoch": 0.20485175202156333,
196
+ "grad_norm": 0.75,
197
+ "learning_rate": 0.001,
198
+ "loss": 3.869609069824219,
199
+ "step": 380
200
+ },
201
+ {
202
+ "epoch": 0.215633423180593,
203
+ "grad_norm": 0.87109375,
204
+ "learning_rate": 0.001,
205
+ "loss": 3.794053649902344,
206
+ "step": 400
207
+ },
208
+ {
209
+ "epoch": 0.215633423180593,
210
+ "eval_loss": 3.7661120891571045,
211
+ "eval_runtime": 10.8481,
212
+ "eval_samples_per_second": 878.215,
213
+ "eval_steps_per_second": 1.751,
214
+ "step": 400
215
+ },
216
+ {
217
+ "epoch": 0.22641509433962265,
218
+ "grad_norm": 0.41015625,
219
+ "learning_rate": 0.001,
220
+ "loss": 3.7193328857421877,
221
+ "step": 420
222
+ },
223
+ {
224
+ "epoch": 0.2371967654986523,
225
+ "grad_norm": 0.69921875,
226
+ "learning_rate": 0.001,
227
+ "loss": 3.6806709289550783,
228
+ "step": 440
229
+ },
230
+ {
231
+ "epoch": 0.24258760107816713,
232
+ "eval_loss": 3.6146388053894043,
233
+ "eval_runtime": 10.8871,
234
+ "eval_samples_per_second": 875.07,
235
+ "eval_steps_per_second": 1.745,
236
+ "step": 450
237
+ },
238
+ {
239
+ "epoch": 0.24797843665768193,
240
+ "grad_norm": 0.2490234375,
241
+ "learning_rate": 0.001,
242
+ "loss": 3.611818313598633,
243
+ "step": 460
244
+ },
245
+ {
246
+ "epoch": 0.2587601078167116,
247
+ "grad_norm": 0.71875,
248
+ "learning_rate": 0.001,
249
+ "loss": 3.5521469116210938,
250
+ "step": 480
251
+ },
252
+ {
253
+ "epoch": 0.2695417789757412,
254
+ "grad_norm": 0.36328125,
255
+ "learning_rate": 0.001,
256
+ "loss": 3.5234119415283205,
257
+ "step": 500
258
+ },
259
+ {
260
+ "epoch": 0.2695417789757412,
261
+ "eval_loss": 3.4891552925109863,
262
+ "eval_runtime": 11.1708,
263
+ "eval_samples_per_second": 852.847,
264
+ "eval_steps_per_second": 1.701,
265
+ "step": 500
266
+ },
267
+ {
268
+ "epoch": 0.2803234501347709,
269
+ "grad_norm": 1.03125,
270
+ "learning_rate": 0.001,
271
+ "loss": 3.472882080078125,
272
+ "step": 520
273
+ },
274
+ {
275
+ "epoch": 0.29110512129380056,
276
+ "grad_norm": 0.337890625,
277
+ "learning_rate": 0.001,
278
+ "loss": 3.431160736083984,
279
+ "step": 540
280
+ },
281
+ {
282
+ "epoch": 0.29649595687331537,
283
+ "eval_loss": 3.376826286315918,
284
+ "eval_runtime": 10.9238,
285
+ "eval_samples_per_second": 872.131,
286
+ "eval_steps_per_second": 1.739,
287
+ "step": 550
288
+ },
289
+ {
290
+ "epoch": 0.3018867924528302,
291
+ "grad_norm": 0.98046875,
292
+ "learning_rate": 0.001,
293
+ "loss": 3.3900474548339843,
294
+ "step": 560
295
+ },
296
+ {
297
+ "epoch": 0.31266846361185985,
298
+ "grad_norm": 0.43359375,
299
+ "learning_rate": 0.001,
300
+ "loss": 3.363066864013672,
301
+ "step": 580
302
+ },
303
+ {
304
+ "epoch": 0.32345013477088946,
305
+ "grad_norm": 0.8125,
306
+ "learning_rate": 0.001,
307
+ "loss": 3.3175891876220702,
308
+ "step": 600
309
+ },
310
+ {
311
+ "epoch": 0.32345013477088946,
312
+ "eval_loss": 3.3390674591064453,
313
+ "eval_runtime": 10.9481,
314
+ "eval_samples_per_second": 870.199,
315
+ "eval_steps_per_second": 1.735,
316
+ "step": 600
317
+ },
318
+ {
319
+ "epoch": 0.33423180592991913,
320
+ "grad_norm": 0.279296875,
321
+ "learning_rate": 0.001,
322
+ "loss": 3.286028289794922,
323
+ "step": 620
324
+ },
325
+ {
326
+ "epoch": 0.3450134770889488,
327
+ "grad_norm": 1.4921875,
328
+ "learning_rate": 0.001,
329
+ "loss": 3.272435760498047,
330
+ "step": 640
331
+ },
332
+ {
333
+ "epoch": 0.3504043126684636,
334
+ "eval_loss": 3.230300188064575,
335
+ "eval_runtime": 10.9925,
336
+ "eval_samples_per_second": 866.683,
337
+ "eval_steps_per_second": 1.728,
338
+ "step": 650
339
+ },
340
+ {
341
+ "epoch": 0.3557951482479784,
342
+ "grad_norm": 0.279296875,
343
+ "learning_rate": 0.001,
344
+ "loss": 3.231139373779297,
345
+ "step": 660
346
+ },
347
+ {
348
+ "epoch": 0.3665768194070081,
349
+ "grad_norm": 1.125,
350
+ "learning_rate": 0.001,
351
+ "loss": 3.191424751281738,
352
+ "step": 680
353
+ },
354
+ {
355
+ "epoch": 0.37735849056603776,
356
+ "grad_norm": 0.59375,
357
+ "learning_rate": 0.001,
358
+ "loss": 3.1910255432128904,
359
+ "step": 700
360
+ },
361
+ {
362
+ "epoch": 0.37735849056603776,
363
+ "eval_loss": 3.167717933654785,
364
+ "eval_runtime": 10.9855,
365
+ "eval_samples_per_second": 867.236,
366
+ "eval_steps_per_second": 1.73,
367
+ "step": 700
368
+ },
369
+ {
370
+ "epoch": 0.3881401617250674,
371
+ "grad_norm": 0.29296875,
372
+ "learning_rate": 0.001,
373
+ "loss": 3.1328031539916994,
374
+ "step": 720
375
+ },
376
+ {
377
+ "epoch": 0.39892183288409705,
378
+ "grad_norm": 0.72265625,
379
+ "learning_rate": 0.001,
380
+ "loss": 3.138438034057617,
381
+ "step": 740
382
+ },
383
+ {
384
+ "epoch": 0.40431266846361186,
385
+ "eval_loss": 3.0950660705566406,
386
+ "eval_runtime": 11.2224,
387
+ "eval_samples_per_second": 848.924,
388
+ "eval_steps_per_second": 1.693,
389
+ "step": 750
390
+ },
391
+ {
392
+ "epoch": 0.40970350404312667,
393
+ "grad_norm": 0.37890625,
394
+ "learning_rate": 0.001,
395
+ "loss": 3.094151496887207,
396
+ "step": 760
397
+ },
398
+ {
399
+ "epoch": 0.42048517520215634,
400
+ "grad_norm": 0.71484375,
401
+ "learning_rate": 0.001,
402
+ "loss": 3.0614999771118163,
403
+ "step": 780
404
+ },
405
+ {
406
+ "epoch": 0.431266846361186,
407
+ "grad_norm": 0.33203125,
408
+ "learning_rate": 0.001,
409
+ "loss": 3.0459268569946287,
410
+ "step": 800
411
+ },
412
+ {
413
+ "epoch": 0.431266846361186,
414
+ "eval_loss": 3.022616147994995,
415
+ "eval_runtime": 11.306,
416
+ "eval_samples_per_second": 842.653,
417
+ "eval_steps_per_second": 1.681,
418
+ "step": 800
419
+ },
420
+ {
421
+ "epoch": 0.4420485175202156,
422
+ "grad_norm": 1.28125,
423
+ "learning_rate": 0.001,
424
+ "loss": 3.014567565917969,
425
+ "step": 820
426
+ },
427
+ {
428
+ "epoch": 0.4528301886792453,
429
+ "grad_norm": 0.37109375,
430
+ "learning_rate": 0.001,
431
+ "loss": 3.010589027404785,
432
+ "step": 840
433
+ },
434
+ {
435
+ "epoch": 0.4582210242587601,
436
+ "eval_loss": 2.9586431980133057,
437
+ "eval_runtime": 11.245,
438
+ "eval_samples_per_second": 847.224,
439
+ "eval_steps_per_second": 1.69,
440
+ "step": 850
441
+ },
442
+ {
443
+ "epoch": 0.4636118598382749,
444
+ "grad_norm": 0.3203125,
445
+ "learning_rate": 0.001,
446
+ "loss": 2.95196533203125,
447
+ "step": 860
448
+ },
449
+ {
450
+ "epoch": 0.4743935309973046,
451
+ "grad_norm": 0.65234375,
452
+ "learning_rate": 0.001,
453
+ "loss": 2.978526496887207,
454
+ "step": 880
455
+ },
456
+ {
457
+ "epoch": 0.48517520215633425,
458
+ "grad_norm": 0.29296875,
459
+ "learning_rate": 0.001,
460
+ "loss": 2.928969955444336,
461
+ "step": 900
462
+ },
463
+ {
464
+ "epoch": 0.48517520215633425,
465
+ "eval_loss": 2.907942056655884,
466
+ "eval_runtime": 11.2684,
467
+ "eval_samples_per_second": 845.463,
468
+ "eval_steps_per_second": 1.686,
469
+ "step": 900
470
+ },
471
+ {
472
+ "epoch": 0.49595687331536387,
473
+ "grad_norm": 1.015625,
474
+ "learning_rate": 0.001,
475
+ "loss": 2.8894153594970704,
476
+ "step": 920
477
+ },
478
+ {
479
+ "epoch": 0.5067385444743935,
480
+ "grad_norm": 0.310546875,
481
+ "learning_rate": 0.001,
482
+ "loss": 2.893274688720703,
483
+ "step": 940
484
+ },
485
+ {
486
+ "epoch": 0.5121293800539084,
487
+ "eval_loss": 2.8531951904296875,
488
+ "eval_runtime": 11.2781,
489
+ "eval_samples_per_second": 844.735,
490
+ "eval_steps_per_second": 1.685,
491
+ "step": 950
492
+ },
493
+ {
494
+ "epoch": 0.5175202156334232,
495
+ "grad_norm": 0.494140625,
496
+ "learning_rate": 0.001,
497
+ "loss": 2.8560728073120116,
498
+ "step": 960
499
+ },
500
+ {
501
+ "epoch": 0.5283018867924528,
502
+ "grad_norm": 0.78515625,
503
+ "learning_rate": 0.001,
504
+ "loss": 2.8492759704589843,
505
+ "step": 980
506
+ },
507
+ {
508
+ "epoch": 0.5390835579514824,
509
+ "grad_norm": 0.453125,
510
+ "learning_rate": 0.001,
511
+ "loss": 2.835124206542969,
512
+ "step": 1000
513
+ },
514
+ {
515
+ "epoch": 0.5390835579514824,
516
+ "eval_loss": 2.8152554035186768,
517
+ "eval_runtime": 11.423,
518
+ "eval_samples_per_second": 834.016,
519
+ "eval_steps_per_second": 1.663,
520
+ "step": 1000
521
+ }
522
+ ],
523
+ "logging_steps": 20,
524
+ "max_steps": 1000,
525
+ "num_input_tokens_seen": 0,
526
+ "num_train_epochs": 1,
527
+ "save_steps": 100,
528
+ "stateful_callbacks": {
529
+ "TrainerControl": {
530
+ "args": {
531
+ "should_epoch_stop": false,
532
+ "should_evaluate": false,
533
+ "should_log": false,
534
+ "should_save": true,
535
+ "should_training_stop": true
536
+ },
537
+ "attributes": {}
538
+ }
539
+ },
540
+ "total_flos": 1.1872833110016e+16,
541
+ "train_batch_size": 512,
542
+ "trial_name": null,
543
+ "trial_params": null
544
+ }
out/mlp-linear-46L_run/checkpoint-1000/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a560522de5d1f6fee179c6909f9239eea63d45e591dc18243e116c36963af413
3
+ size 4920
out/mlp-linear-46L_run/checkpoint-200/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "linear",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 46,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-linear-46L_run/checkpoint-200/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a6745aedc72f76d3ff27848c6fd769cf26823c4f448ae47141989348026bbbd
3
+ size 16186168
out/mlp-linear-46L_run/checkpoint-200/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99ab1485ddd9bb0b9e428c55b47767c0b6d687859a12c45d05a10a206b3e2f4c
3
+ size 32605416
out/mlp-linear-46L_run/checkpoint-200/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecefbb3f17bb76b6655eb0157c98b5287c17fa4b4c72a6b9068b0823ce9fd18d
3
+ size 14244
out/mlp-linear-46L_run/checkpoint-200/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d3b5a970fc92af42664e84ba686059eaaacfaa668691c0cf6d2afddda677461
3
+ size 1064
out/mlp-linear-46L_run/checkpoint-200/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-linear-46L_run/checkpoint-200/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-linear-46L_run/checkpoint-200/trainer_state.json ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.1078167115902965,
6
+ "eval_steps": 50,
7
+ "global_step": 200,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9921875,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.2093955993652346,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.337890625,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.994318008422852,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.666038990020752,
29
+ "eval_runtime": 10.8497,
30
+ "eval_samples_per_second": 878.09,
31
+ "eval_steps_per_second": 1.751,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.6640625,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.679982757568359,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 0.4921875,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.49438705444336,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 1.3125,
51
+ "learning_rate": 0.001,
52
+ "loss": 5.421606826782226,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 5.367257118225098,
58
+ "eval_runtime": 10.9702,
59
+ "eval_samples_per_second": 868.441,
60
+ "eval_steps_per_second": 1.732,
61
+ "step": 100
62
+ },
63
+ {
64
+ "epoch": 0.0646900269541779,
65
+ "grad_norm": 1.1640625,
66
+ "learning_rate": 0.001,
67
+ "loss": 5.309521102905274,
68
+ "step": 120
69
+ },
70
+ {
71
+ "epoch": 0.07547169811320754,
72
+ "grad_norm": 1.4140625,
73
+ "learning_rate": 0.001,
74
+ "loss": 5.126371765136719,
75
+ "step": 140
76
+ },
77
+ {
78
+ "epoch": 0.08086253369272237,
79
+ "eval_loss": 4.993337154388428,
80
+ "eval_runtime": 10.9852,
81
+ "eval_samples_per_second": 867.26,
82
+ "eval_steps_per_second": 1.73,
83
+ "step": 150
84
+ },
85
+ {
86
+ "epoch": 0.0862533692722372,
87
+ "grad_norm": 1.5078125,
88
+ "learning_rate": 0.001,
89
+ "loss": 4.988518524169922,
90
+ "step": 160
91
+ },
92
+ {
93
+ "epoch": 0.09703504043126684,
94
+ "grad_norm": 1.8828125,
95
+ "learning_rate": 0.001,
96
+ "loss": 4.8510185241699215,
97
+ "step": 180
98
+ },
99
+ {
100
+ "epoch": 0.1078167115902965,
101
+ "grad_norm": 1.28125,
102
+ "learning_rate": 0.001,
103
+ "loss": 4.724799346923828,
104
+ "step": 200
105
+ },
106
+ {
107
+ "epoch": 0.1078167115902965,
108
+ "eval_loss": 4.646933555603027,
109
+ "eval_runtime": 10.9059,
110
+ "eval_samples_per_second": 873.563,
111
+ "eval_steps_per_second": 1.742,
112
+ "step": 200
113
+ }
114
+ ],
115
+ "logging_steps": 20,
116
+ "max_steps": 1000,
117
+ "num_input_tokens_seen": 0,
118
+ "num_train_epochs": 1,
119
+ "save_steps": 100,
120
+ "stateful_callbacks": {
121
+ "TrainerControl": {
122
+ "args": {
123
+ "should_epoch_stop": false,
124
+ "should_evaluate": false,
125
+ "should_log": false,
126
+ "should_save": true,
127
+ "should_training_stop": false
128
+ },
129
+ "attributes": {}
130
+ }
131
+ },
132
+ "total_flos": 2374566622003200.0,
133
+ "train_batch_size": 512,
134
+ "trial_name": null,
135
+ "trial_params": null
136
+ }
out/mlp-linear-46L_run/checkpoint-200/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a560522de5d1f6fee179c6909f9239eea63d45e591dc18243e116c36963af413
3
+ size 4920
out/mlp-linear-46L_run/checkpoint-300/config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "linear",
3
+ "architectures": [
4
+ "TinyLlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "dtype": "bfloat16",
10
+ "eos_token_id": 2,
11
+ "head_dim": 32,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 128,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 256,
16
+ "max_position_embeddings": 512,
17
+ "mlp_bias": false,
18
+ "mlp_type": "mlp",
19
+ "model_type": "tiny_llama",
20
+ "num_attention_heads": 4,
21
+ "num_hidden_layers": 46,
22
+ "num_key_value_heads": 4,
23
+ "pad_token_id": 0,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-06,
26
+ "rope_parameters": {
27
+ "rope_theta": 10000.0,
28
+ "rope_type": "default"
29
+ },
30
+ "tie_word_embeddings": true,
31
+ "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
32
+ "transformers_version": "5.15.0.dev0",
33
+ "use_cache": false,
34
+ "vocab_size": 4096
35
+ }
out/mlp-linear-46L_run/checkpoint-300/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24dd236bf84952e5d1f0b207ed16dadd1bc45982965b18d31b8b11d5db15ade4
3
+ size 16186168
out/mlp-linear-46L_run/checkpoint-300/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5b27a610a9f4cb2f91bda3cd8aa59d06ca682c8aa6f30971332cf6e0a113d3f
3
+ size 32605416
out/mlp-linear-46L_run/checkpoint-300/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:068fbd993087219c15b8c0baa13fc39644a4dcdfe92d8be3fa6434deece90371
3
+ size 14244
out/mlp-linear-46L_run/checkpoint-300/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76cd21f32221b90fb8288fd2428595ec8b5039fe2425086d67cacb5c8d814c9e
3
+ size 1064
out/mlp-linear-46L_run/checkpoint-300/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
out/mlp-linear-46L_run/checkpoint-300/tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<|endoftext|>",
5
+ "eos_token": "<|endoftext|>",
6
+ "errors": "replace",
7
+ "is_local": false,
8
+ "local_files_only": false,
9
+ "model_max_length": 1024,
10
+ "pad_token": "<|endoftext|>",
11
+ "tokenizer_class": "GPT2Tokenizer",
12
+ "unk_token": "<|endoftext|>"
13
+ }
out/mlp-linear-46L_run/checkpoint-300/trainer_state.json ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.16172506738544473,
6
+ "eval_steps": 50,
7
+ "global_step": 300,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.01078167115902965,
14
+ "grad_norm": 0.9921875,
15
+ "learning_rate": 0.001,
16
+ "loss": 7.2093955993652346,
17
+ "step": 20
18
+ },
19
+ {
20
+ "epoch": 0.0215633423180593,
21
+ "grad_norm": 0.337890625,
22
+ "learning_rate": 0.001,
23
+ "loss": 5.994318008422852,
24
+ "step": 40
25
+ },
26
+ {
27
+ "epoch": 0.026954177897574125,
28
+ "eval_loss": 5.666038990020752,
29
+ "eval_runtime": 10.8497,
30
+ "eval_samples_per_second": 878.09,
31
+ "eval_steps_per_second": 1.751,
32
+ "step": 50
33
+ },
34
+ {
35
+ "epoch": 0.03234501347708895,
36
+ "grad_norm": 0.6640625,
37
+ "learning_rate": 0.001,
38
+ "loss": 5.679982757568359,
39
+ "step": 60
40
+ },
41
+ {
42
+ "epoch": 0.0431266846361186,
43
+ "grad_norm": 0.4921875,
44
+ "learning_rate": 0.001,
45
+ "loss": 5.49438705444336,
46
+ "step": 80
47
+ },
48
+ {
49
+ "epoch": 0.05390835579514825,
50
+ "grad_norm": 1.3125,
51
+ "learning_rate": 0.001,
52
+ "loss": 5.421606826782226,
53
+ "step": 100
54
+ },
55
+ {
56
+ "epoch": 0.05390835579514825,
57
+ "eval_loss": 5.367257118225098,
58
+ "eval_runtime": 10.9702,
59
+ "eval_samples_per_second": 868.441,
60
+ "eval_steps_per_second": 1.732,
61
+ "step": 100
62
+ },
63
+ {
64
+ "epoch": 0.0646900269541779,
65
+ "grad_norm": 1.1640625,
66
+ "learning_rate": 0.001,
67
+ "loss": 5.309521102905274,
68
+ "step": 120
69
+ },
70
+ {
71
+ "epoch": 0.07547169811320754,
72
+ "grad_norm": 1.4140625,
73
+ "learning_rate": 0.001,
74
+ "loss": 5.126371765136719,
75
+ "step": 140
76
+ },
77
+ {
78
+ "epoch": 0.08086253369272237,
79
+ "eval_loss": 4.993337154388428,
80
+ "eval_runtime": 10.9852,
81
+ "eval_samples_per_second": 867.26,
82
+ "eval_steps_per_second": 1.73,
83
+ "step": 150
84
+ },
85
+ {
86
+ "epoch": 0.0862533692722372,
87
+ "grad_norm": 1.5078125,
88
+ "learning_rate": 0.001,
89
+ "loss": 4.988518524169922,
90
+ "step": 160
91
+ },
92
+ {
93
+ "epoch": 0.09703504043126684,
94
+ "grad_norm": 1.8828125,
95
+ "learning_rate": 0.001,
96
+ "loss": 4.8510185241699215,
97
+ "step": 180
98
+ },
99
+ {
100
+ "epoch": 0.1078167115902965,
101
+ "grad_norm": 1.28125,
102
+ "learning_rate": 0.001,
103
+ "loss": 4.724799346923828,
104
+ "step": 200
105
+ },
106
+ {
107
+ "epoch": 0.1078167115902965,
108
+ "eval_loss": 4.646933555603027,
109
+ "eval_runtime": 10.9059,
110
+ "eval_samples_per_second": 873.563,
111
+ "eval_steps_per_second": 1.742,
112
+ "step": 200
113
+ },
114
+ {
115
+ "epoch": 0.11859838274932614,
116
+ "grad_norm": 3.03125,
117
+ "learning_rate": 0.001,
118
+ "loss": 4.576333618164062,
119
+ "step": 220
120
+ },
121
+ {
122
+ "epoch": 0.1293800539083558,
123
+ "grad_norm": 3.28125,
124
+ "learning_rate": 0.001,
125
+ "loss": 4.479412841796875,
126
+ "step": 240
127
+ },
128
+ {
129
+ "epoch": 0.1347708894878706,
130
+ "eval_loss": 4.414884567260742,
131
+ "eval_runtime": 11.1112,
132
+ "eval_samples_per_second": 857.424,
133
+ "eval_steps_per_second": 1.71,
134
+ "step": 250
135
+ },
136
+ {
137
+ "epoch": 0.14016172506738545,
138
+ "grad_norm": 1.703125,
139
+ "learning_rate": 0.001,
140
+ "loss": 4.398517608642578,
141
+ "step": 260
142
+ },
143
+ {
144
+ "epoch": 0.1509433962264151,
145
+ "grad_norm": 0.416015625,
146
+ "learning_rate": 0.001,
147
+ "loss": 4.271953582763672,
148
+ "step": 280
149
+ },
150
+ {
151
+ "epoch": 0.16172506738544473,
152
+ "grad_norm": 0.63671875,
153
+ "learning_rate": 0.001,
154
+ "loss": 4.192776870727539,
155
+ "step": 300
156
+ },
157
+ {
158
+ "epoch": 0.16172506738544473,
159
+ "eval_loss": 4.156349182128906,
160
+ "eval_runtime": 10.9589,
161
+ "eval_samples_per_second": 869.336,
162
+ "eval_steps_per_second": 1.734,
163
+ "step": 300
164
+ }
165
+ ],
166
+ "logging_steps": 20,
167
+ "max_steps": 1000,
168
+ "num_input_tokens_seen": 0,
169
+ "num_train_epochs": 1,
170
+ "save_steps": 100,
171
+ "stateful_callbacks": {
172
+ "TrainerControl": {
173
+ "args": {
174
+ "should_epoch_stop": false,
175
+ "should_evaluate": false,
176
+ "should_log": false,
177
+ "should_save": true,
178
+ "should_training_stop": false
179
+ },
180
+ "attributes": {}
181
+ }
182
+ },
183
+ "total_flos": 3561849933004800.0,
184
+ "train_batch_size": 512,
185
+ "trial_name": null,
186
+ "trial_params": null
187
+ }