Auto upload 2026-08-06T22:52:15.423327

out/glu-gelu-94L_run/checkpoint-1000/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 94,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-gelu-94L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6223bdad6880416e4ed61d165683234d0948eb9ed3b73ccbf5e19234128785a2
+size 31992144

out/glu-gelu-94L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32f9feb240e6b16414d7d776d8732520089a9e8c7635563c07c785c49974b591
+size 64508674

out/glu-gelu-94L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b6047bd8cc6f4cdf7c46dea47edb8e542435510070c6cd1e0a7d9ccf5fd7da
+size 14244

out/glu-gelu-94L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5f71c5fa69d56807b14256b78700be6088fcb1bda0ff8e0ea6f8709a7df2fde
+size 1064

out/glu-gelu-94L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

out/glu-gelu-94L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,424 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.06740815638692282,
+  "eval_steps": 200,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.25,
+      "learning_rate": 0.0003,
+      "loss": 7.761959075927734,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.15625,
+      "learning_rate": 0.0003,
+      "loss": 7.0005943298339846,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.8984375,
+      "learning_rate": 0.0003,
+      "loss": 6.469890594482422,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.55859375,
+      "learning_rate": 0.0003,
+      "loss": 6.16566276550293,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003,
+      "loss": 6.043316650390625,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.40234375,
+      "learning_rate": 0.0003,
+      "loss": 5.951222610473633,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.0003,
+      "loss": 5.8204914093017575,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 1.9453125,
+      "learning_rate": 0.0003,
+      "loss": 5.5939079284667965,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0003,
+      "loss": 5.211606979370117,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.0003,
+      "loss": 4.872050094604492,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 4.710168361663818,
+      "eval_runtime": 17.0059,
+      "eval_samples_per_second": 560.218,
+      "eval_steps_per_second": 8.762,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 2.859375,
+      "learning_rate": 0.0003,
+      "loss": 4.602034378051758,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 4.0,
+      "learning_rate": 0.0003,
+      "loss": 4.388680267333984,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.3203125,
+      "learning_rate": 0.0003,
+      "loss": 4.151194381713867,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 1.125,
+      "learning_rate": 0.0003,
+      "loss": 3.8524208068847656,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.4453125,
+      "learning_rate": 0.0003,
+      "loss": 3.514937973022461,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 1.09375,
+      "learning_rate": 0.0003,
+      "loss": 3.273800277709961,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 1.03125,
+      "learning_rate": 0.0003,
+      "loss": 3.068279266357422,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.6640625,
+      "learning_rate": 0.0003,
+      "loss": 2.8562135696411133,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 2.53125,
+      "learning_rate": 0.0003,
+      "loss": 2.6943243026733397,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 2.109375,
+      "learning_rate": 0.0003,
+      "loss": 2.540350914001465,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 2.481466770172119,
+      "eval_runtime": 17.0774,
+      "eval_samples_per_second": 557.871,
+      "eval_steps_per_second": 8.725,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 2.3923057556152343,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 1.640625,
+      "learning_rate": 0.0003,
+      "loss": 2.2704214096069335,
+      "step": 440
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 1.5546875,
+      "learning_rate": 0.0003,
+      "loss": 2.1233325958251954,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 1.578125,
+      "learning_rate": 0.0003,
+      "loss": 1.9938255310058595,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 1.5,
+      "learning_rate": 0.0003,
+      "loss": 1.890728759765625,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 1.90625,
+      "learning_rate": 0.0003,
+      "loss": 1.8091728210449218,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 2.265625,
+      "learning_rate": 0.0003,
+      "loss": 1.7210920333862305,
+      "step": 540
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 1.421875,
+      "learning_rate": 0.0003,
+      "loss": 1.6450265884399413,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 1.4296875,
+      "learning_rate": 0.0003,
+      "loss": 1.5685007095336914,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 1.34375,
+      "learning_rate": 0.0003,
+      "loss": 1.4878437042236328,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 1.4193751811981201,
+      "eval_runtime": 17.0783,
+      "eval_samples_per_second": 557.844,
+      "eval_steps_per_second": 8.725,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.609375,
+      "learning_rate": 0.0003,
+      "loss": 1.360587978363037,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 2.28125,
+      "learning_rate": 0.0003,
+      "loss": 1.3004804611206056,
+      "step": 640
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 1.453125,
+      "learning_rate": 0.0003,
+      "loss": 1.2486857414245605,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 1.0390625,
+      "learning_rate": 0.0003,
+      "loss": 1.2126808166503906,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 1.1561152458190918,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0003,
+      "loss": 1.107057476043701,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 2.375,
+      "learning_rate": 0.0003,
+      "loss": 1.0661657333374024,
+      "step": 740
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 1.6015625,
+      "learning_rate": 0.0003,
+      "loss": 1.0436604499816895,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 1.140625,
+      "learning_rate": 0.0003,
+      "loss": 1.0305798530578614,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.40625,
+      "learning_rate": 0.0003,
+      "loss": 0.970916748046875,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 0.9400641322135925,
+      "eval_runtime": 17.019,
+      "eval_samples_per_second": 559.786,
+      "eval_steps_per_second": 8.755,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.0003,
+      "loss": 0.9162347793579102,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.310546875,
+      "learning_rate": 0.0003,
+      "loss": 0.8616827964782715,
+      "step": 840
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.3984375,
+      "learning_rate": 0.0003,
+      "loss": 0.8070379257202148,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0003,
+      "loss": 0.7639906406402588,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 2.65625,
+      "learning_rate": 0.0003,
+      "loss": 0.8199039459228515,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.953125,
+      "learning_rate": 0.0003,
+      "loss": 0.7552119255065918,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.5078125,
+      "learning_rate": 0.0003,
+      "loss": 0.7163666725158692,
+      "step": 940
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.33984375,
+      "learning_rate": 0.0003,
+      "loss": 0.6772623538970948,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0003,
+      "loss": 0.6524566173553467,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.271484375,
+      "learning_rate": 0.0003,
+      "loss": 0.6347129821777344,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 0.619852602481842,
+      "eval_runtime": 17.1493,
+      "eval_samples_per_second": 555.533,
+      "eval_steps_per_second": 8.688,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3032708284416000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

out/glu-gelu-94L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8d0278b53431342736bcbfc1a79e8570a187db4fbd922dcaa762a3ab2e1207a
+size 4920

out/glu-gelu-94L_run/checkpoint-500/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 94,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-gelu-94L_run/checkpoint-500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df1f4766eace42c3ae0e62b5b237b8f5742f523ed6fae5d313808b672e8a5ad2
+size 31992144

out/glu-gelu-94L_run/checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2536e18f5e731624e72e74c3f466fdfcde5e3aef0b49887126b153f342618a55
+size 64508674

out/glu-gelu-94L_run/checkpoint-500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf9097d4513154245c48236b6ec5137b7ee2a21c9f58f2cba798ea275c6026f
+size 14244

out/glu-gelu-94L_run/checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e328dcb5672eefa48692126747c226aede5ae2e575011f8f5a2b20326a61f850
+size 1064

out/glu-gelu-94L_run/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

out/glu-gelu-94L_run/checkpoint-500/trainer_state.json

@@ -0,0 +1,225 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.03370407819346141,
+  "eval_steps": 200,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.25,
+      "learning_rate": 0.0003,
+      "loss": 7.761959075927734,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.15625,
+      "learning_rate": 0.0003,
+      "loss": 7.0005943298339846,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.8984375,
+      "learning_rate": 0.0003,
+      "loss": 6.469890594482422,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.55859375,
+      "learning_rate": 0.0003,
+      "loss": 6.16566276550293,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003,
+      "loss": 6.043316650390625,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.40234375,
+      "learning_rate": 0.0003,
+      "loss": 5.951222610473633,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 2.1875,
+      "learning_rate": 0.0003,
+      "loss": 5.8204914093017575,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 1.9453125,
+      "learning_rate": 0.0003,
+      "loss": 5.5939079284667965,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.0003,
+      "loss": 5.211606979370117,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.69921875,
+      "learning_rate": 0.0003,
+      "loss": 4.872050094604492,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 4.710168361663818,
+      "eval_runtime": 17.0059,
+      "eval_samples_per_second": 560.218,
+      "eval_steps_per_second": 8.762,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 2.859375,
+      "learning_rate": 0.0003,
+      "loss": 4.602034378051758,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 4.0,
+      "learning_rate": 0.0003,
+      "loss": 4.388680267333984,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.3203125,
+      "learning_rate": 0.0003,
+      "loss": 4.151194381713867,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 1.125,
+      "learning_rate": 0.0003,
+      "loss": 3.8524208068847656,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.4453125,
+      "learning_rate": 0.0003,
+      "loss": 3.514937973022461,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 1.09375,
+      "learning_rate": 0.0003,
+      "loss": 3.273800277709961,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 1.03125,
+      "learning_rate": 0.0003,
+      "loss": 3.068279266357422,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.6640625,
+      "learning_rate": 0.0003,
+      "loss": 2.8562135696411133,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 2.53125,
+      "learning_rate": 0.0003,
+      "loss": 2.6943243026733397,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 2.109375,
+      "learning_rate": 0.0003,
+      "loss": 2.540350914001465,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 2.481466770172119,
+      "eval_runtime": 17.0774,
+      "eval_samples_per_second": 557.871,
+      "eval_steps_per_second": 8.725,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 2.3923057556152343,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 1.640625,
+      "learning_rate": 0.0003,
+      "loss": 2.2704214096069335,
+      "step": 440
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 1.5546875,
+      "learning_rate": 0.0003,
+      "loss": 2.1233325958251954,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 1.578125,
+      "learning_rate": 0.0003,
+      "loss": 1.9938255310058595,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 1.5,
+      "learning_rate": 0.0003,
+      "loss": 1.890728759765625,
+      "step": 500
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1516354142208000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

out/glu-gelu-94L_run/checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8d0278b53431342736bcbfc1a79e8570a187db4fbd922dcaa762a3ab2e1207a
+size 4920

out/glu-gelu-94L_run/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 94,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-gelu-94L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6223bdad6880416e4ed61d165683234d0948eb9ed3b73ccbf5e19234128785a2
+size 31992144

out/glu-gelu-94L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8d0278b53431342736bcbfc1a79e8570a187db4fbd922dcaa762a3ab2e1207a
+size 4920