Auto upload 2026-08-06T21:25:44.036029

out/glu-s10-21L_run/checkpoint-500/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "s10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-s10-21L_run/checkpoint-500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9188311675f967f98c10c6675503f8b0725dfec81e9b5c02a5de9de5646cf95
+size 7961528

out/glu-s10-21L_run/checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dca663ec18d49680dfd5c66f637ba3f8cd60e7837e7169e5fed2f5420aa1e6e2
+size 16041274

out/glu-s10-21L_run/checkpoint-500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf9097d4513154245c48236b6ec5137b7ee2a21c9f58f2cba798ea275c6026f
+size 14244

out/glu-s10-21L_run/checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e328dcb5672eefa48692126747c226aede5ae2e575011f8f5a2b20326a61f850
+size 1064

out/glu-s10-21L_run/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

out/glu-s10-21L_run/checkpoint-500/trainer_state.json

@@ -0,0 +1,225 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.03370407819346141,
+  "eval_steps": 200,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.265625,
+      "learning_rate": 0.0003,
+      "loss": 7.819881439208984,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 7.0600433349609375,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.98828125,
+      "learning_rate": 0.0003,
+      "loss": 6.507986450195313,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.0703125,
+      "learning_rate": 0.0003,
+      "loss": 6.13072395324707,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 1.8359375,
+      "learning_rate": 0.0003,
+      "loss": 5.789023208618164,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.796875,
+      "learning_rate": 0.0003,
+      "loss": 5.358802795410156,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 2.46875,
+      "learning_rate": 0.0003,
+      "loss": 4.854774856567383,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 2.671875,
+      "learning_rate": 0.0003,
+      "loss": 4.344797134399414,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 2.453125,
+      "learning_rate": 0.0003,
+      "loss": 3.923533248901367,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 2.90625,
+      "learning_rate": 0.0003,
+      "loss": 3.5441238403320314,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 3.3532824516296387,
+      "eval_runtime": 9.7034,
+      "eval_samples_per_second": 981.821,
+      "eval_steps_per_second": 15.355,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 2.703125,
+      "learning_rate": 0.0003,
+      "loss": 3.1950939178466795,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.65625,
+      "learning_rate": 0.0003,
+      "loss": 2.851460075378418,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.6953125,
+      "learning_rate": 0.0003,
+      "loss": 2.524063491821289,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 1.71875,
+      "learning_rate": 0.0003,
+      "loss": 2.1918392181396484,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.734375,
+      "learning_rate": 0.0003,
+      "loss": 1.8681827545166017,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 1.3515625,
+      "learning_rate": 0.0003,
+      "loss": 1.5638505935668945,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.0003,
+      "loss": 1.3047816276550293,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.1484375,
+      "learning_rate": 0.0003,
+      "loss": 1.1159173011779786,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.61328125,
+      "learning_rate": 0.0003,
+      "loss": 0.9435104370117188,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.4609375,
+      "learning_rate": 0.0003,
+      "loss": 0.7958864688873291,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 0.7467897534370422,
+      "eval_runtime": 9.7733,
+      "eval_samples_per_second": 974.795,
+      "eval_steps_per_second": 15.246,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.43359375,
+      "learning_rate": 0.0003,
+      "loss": 0.6977367401123047,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.0003,
+      "loss": 0.6431714534759522,
+      "step": 440
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.0003,
+      "loss": 0.5860850811004639,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.455078125,
+      "learning_rate": 0.0003,
+      "loss": 0.5521106243133544,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.0003,
+      "loss": 0.5192345142364502,
+      "step": 500
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 338769739776000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

out/glu-s10-21L_run/checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:750ae5940581a03010874ce9e4b78ed370c49298da11e5fc09733ce113a086fd
+size 4920

out/glu-s10-21L_run/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "s10",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-s10-21L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9188311675f967f98c10c6675503f8b0725dfec81e9b5c02a5de9de5646cf95
+size 7961528

out/glu-s10-21L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:750ae5940581a03010874ce9e4b78ed370c49298da11e5fc09733ce113a086fd
+size 4920

out/glu-situglu-21L_run/README.md

@@ -0,0 +1,65 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+model-index:
+- name: ACTio-glu-situglu-21L
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# ACTio-glu-situglu-21L
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2260
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 64
+- eval_batch_size: 64
+- seed: 42
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: constant
+- training_steps: 2000
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 3.5706        | 0.0135 | 200  | 3.3888          |
+| 1.0316        | 0.0270 | 400  | 0.9727          |
+| 0.4497        | 0.0404 | 600  | 0.4427          |
+| 0.3517        | 0.0539 | 800  | 0.3524          |
+| 0.3066        | 0.0674 | 1000 | 0.3055          |
+| 0.2826        | 0.0809 | 1200 | 0.2804          |
+| 0.2555        | 0.0944 | 1400 | 0.2571          |
+| 0.2742        | 0.1079 | 1600 | 0.2722          |
+| 0.2340        | 0.1213 | 1800 | 0.2329          |
+| 0.2245        | 0.1348 | 2000 | 0.2260          |
+
+
+### Framework versions
+
+- Transformers 5.15.0.dev0
+- Pytorch 2.6.0+cu124
+- Datasets 5.0.1
+- Tokenizers 0.22.2

out/glu-situglu-21L_run/checkpoint-1000/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "situglu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-situglu-21L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41ce491ca031ddb2b619f9ed5889f7ed52321f2a30bc07125e2e03b9e36622fc
+size 7961656

out/glu-situglu-21L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b729dba4d2dc991d1e9f653b39772fe3864b0455443caa61d350bc2537e496
+size 16041274

out/glu-situglu-21L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b6047bd8cc6f4cdf7c46dea47edb8e542435510070c6cd1e0a7d9ccf5fd7da
+size 14244

out/glu-situglu-21L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5f71c5fa69d56807b14256b78700be6088fcb1bda0ff8e0ea6f8709a7df2fde
+size 1064

out/glu-situglu-21L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

out/glu-situglu-21L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,424 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.06740815638692282,
+  "eval_steps": 200,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.2578125,
+      "learning_rate": 0.0003,
+      "loss": 7.813759613037109,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 7.046382141113281,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.98046875,
+      "learning_rate": 0.0003,
+      "loss": 6.4861198425292965,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.0003,
+      "loss": 6.114296722412109,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 2.375,
+      "learning_rate": 0.0003,
+      "loss": 5.7780506134033205,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.0003,
+      "loss": 5.281050491333008,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 2.953125,
+      "learning_rate": 0.0003,
+      "loss": 4.785240554809571,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 2.921875,
+      "learning_rate": 0.0003,
+      "loss": 4.324177551269531,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 2.65625,
+      "learning_rate": 0.0003,
+      "loss": 3.9370296478271483,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 2.78125,
+      "learning_rate": 0.0003,
+      "loss": 3.5706451416015623,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 3.388801336288452,
+      "eval_runtime": 9.9378,
+      "eval_samples_per_second": 958.66,
+      "eval_steps_per_second": 14.993,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 3.21875,
+      "learning_rate": 0.0003,
+      "loss": 3.235015106201172,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.25,
+      "learning_rate": 0.0003,
+      "loss": 2.9087091445922852,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.0003,
+      "loss": 2.606447219848633,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 2.28125,
+      "learning_rate": 0.0003,
+      "loss": 2.309796142578125,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.5859375,
+      "learning_rate": 0.0003,
+      "loss": 2.038427734375,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 1.65625,
+      "learning_rate": 0.0003,
+      "loss": 1.7724691390991212,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.0003,
+      "loss": 1.5364046096801758,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.0003,
+      "loss": 1.3601617813110352,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 1.203125,
+      "learning_rate": 0.0003,
+      "loss": 1.1919203758239747,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.97265625,
+      "learning_rate": 0.0003,
+      "loss": 1.031599998474121,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 0.9726812839508057,
+      "eval_runtime": 9.9094,
+      "eval_samples_per_second": 961.411,
+      "eval_steps_per_second": 15.036,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 1.078125,
+      "learning_rate": 0.0003,
+      "loss": 0.9072963714599609,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 1.0859375,
+      "learning_rate": 0.0003,
+      "loss": 0.8252955436706543,
+      "step": 440
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0003,
+      "loss": 0.7282839775085449,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.48828125,
+      "learning_rate": 0.0003,
+      "loss": 0.6567337512969971,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.380859375,
+      "learning_rate": 0.0003,
+      "loss": 0.5960001945495605,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0003,
+      "loss": 0.5524660587310791,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0003,
+      "loss": 0.5125324249267578,
+      "step": 540
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003,
+      "loss": 0.4881768226623535,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0003,
+      "loss": 0.463836669921875,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003,
+      "loss": 0.44971308708190916,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 0.4426644444465637,
+      "eval_runtime": 9.9173,
+      "eval_samples_per_second": 960.644,
+      "eval_steps_per_second": 15.024,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0003,
+      "loss": 0.4324655055999756,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0003,
+      "loss": 0.4214168548583984,
+      "step": 640
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.0003,
+      "loss": 0.4324348449707031,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0003,
+      "loss": 0.4121569633483887,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.3203125,
+      "learning_rate": 0.0003,
+      "loss": 0.3938265800476074,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.28515625,
+      "learning_rate": 0.0003,
+      "loss": 0.3844801664352417,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.3729189395904541,
+      "step": 740
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.28125,
+      "learning_rate": 0.0003,
+      "loss": 0.3688415765762329,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.36176743507385256,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003,
+      "loss": 0.3517471790313721,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 0.35235798358917236,
+      "eval_runtime": 10.0631,
+      "eval_samples_per_second": 946.728,
+      "eval_steps_per_second": 14.807,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.34776208400726316,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.0003,
+      "loss": 0.34154040813446046,
+      "step": 840
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.3332885980606079,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.3266940116882324,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.306640625,
+      "learning_rate": 0.0003,
+      "loss": 0.32539618015289307,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.32114400863647463,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.296875,
+      "learning_rate": 0.0003,
+      "loss": 0.31635844707489014,
+      "step": 940
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.0003,
+      "loss": 0.31109464168548584,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.0003,
+      "loss": 0.30814523696899415,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.322265625,
+      "learning_rate": 0.0003,
+      "loss": 0.3065646648406982,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 0.3054979145526886,
+      "eval_runtime": 9.9252,
+      "eval_samples_per_second": 959.884,
+      "eval_steps_per_second": 15.012,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 677539479552000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

out/glu-situglu-21L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711ad909475cdef5e2f24026d199875e77c559083ded851f0b6f7b4ae9dadd26
+size 4920

out/glu-situglu-21L_run/checkpoint-1500/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "situglu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-situglu-21L_run/checkpoint-1500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93e1399215197617ca35ca2f27faf97ee34e1ef27c3d4598f18ca907c6573566
+size 7961656

out/glu-situglu-21L_run/checkpoint-1500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:709995f82c3348151774e9986bf8619392657fecd99ea348c4b5a46b3c9eb43d
+size 16041274

out/glu-situglu-21L_run/checkpoint-1500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2768285b45b2a0c05f6f50bbb8c0287fca6f62a8cde6d1b1f02151ac72ee8dc
+size 14244

out/glu-situglu-21L_run/checkpoint-1500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b26a759cc207e28dea06eae7ad531c2dd78a09978f6972edbfc1566adccd7142
+size 1064

out/glu-situglu-21L_run/checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

out/glu-situglu-21L_run/checkpoint-1500/trainer_state.json

@@ -0,0 +1,615 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.10111223458038422,
+  "eval_steps": 200,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.2578125,
+      "learning_rate": 0.0003,
+      "loss": 7.813759613037109,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 7.046382141113281,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.98046875,
+      "learning_rate": 0.0003,
+      "loss": 6.4861198425292965,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.0003,
+      "loss": 6.114296722412109,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 2.375,
+      "learning_rate": 0.0003,
+      "loss": 5.7780506134033205,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.0003,
+      "loss": 5.281050491333008,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 2.953125,
+      "learning_rate": 0.0003,
+      "loss": 4.785240554809571,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 2.921875,
+      "learning_rate": 0.0003,
+      "loss": 4.324177551269531,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 2.65625,
+      "learning_rate": 0.0003,
+      "loss": 3.9370296478271483,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 2.78125,
+      "learning_rate": 0.0003,
+      "loss": 3.5706451416015623,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 3.388801336288452,
+      "eval_runtime": 9.9378,
+      "eval_samples_per_second": 958.66,
+      "eval_steps_per_second": 14.993,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 3.21875,
+      "learning_rate": 0.0003,
+      "loss": 3.235015106201172,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.25,
+      "learning_rate": 0.0003,
+      "loss": 2.9087091445922852,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.0003,
+      "loss": 2.606447219848633,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 2.28125,
+      "learning_rate": 0.0003,
+      "loss": 2.309796142578125,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.5859375,
+      "learning_rate": 0.0003,
+      "loss": 2.038427734375,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 1.65625,
+      "learning_rate": 0.0003,
+      "loss": 1.7724691390991212,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.0003,
+      "loss": 1.5364046096801758,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.0003,
+      "loss": 1.3601617813110352,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 1.203125,
+      "learning_rate": 0.0003,
+      "loss": 1.1919203758239747,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.97265625,
+      "learning_rate": 0.0003,
+      "loss": 1.031599998474121,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 0.9726812839508057,
+      "eval_runtime": 9.9094,
+      "eval_samples_per_second": 961.411,
+      "eval_steps_per_second": 15.036,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 1.078125,
+      "learning_rate": 0.0003,
+      "loss": 0.9072963714599609,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 1.0859375,
+      "learning_rate": 0.0003,
+      "loss": 0.8252955436706543,
+      "step": 440
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0003,
+      "loss": 0.7282839775085449,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.48828125,
+      "learning_rate": 0.0003,
+      "loss": 0.6567337512969971,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.380859375,
+      "learning_rate": 0.0003,
+      "loss": 0.5960001945495605,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0003,
+      "loss": 0.5524660587310791,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0003,
+      "loss": 0.5125324249267578,
+      "step": 540
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003,
+      "loss": 0.4881768226623535,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0003,
+      "loss": 0.463836669921875,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003,
+      "loss": 0.44971308708190916,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 0.4426644444465637,
+      "eval_runtime": 9.9173,
+      "eval_samples_per_second": 960.644,
+      "eval_steps_per_second": 15.024,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0003,
+      "loss": 0.4324655055999756,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0003,
+      "loss": 0.4214168548583984,
+      "step": 640
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.0003,
+      "loss": 0.4324348449707031,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0003,
+      "loss": 0.4121569633483887,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.3203125,
+      "learning_rate": 0.0003,
+      "loss": 0.3938265800476074,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.28515625,
+      "learning_rate": 0.0003,
+      "loss": 0.3844801664352417,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.3729189395904541,
+      "step": 740
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.28125,
+      "learning_rate": 0.0003,
+      "loss": 0.3688415765762329,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.36176743507385256,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003,
+      "loss": 0.3517471790313721,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 0.35235798358917236,
+      "eval_runtime": 10.0631,
+      "eval_samples_per_second": 946.728,
+      "eval_steps_per_second": 14.807,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.34776208400726316,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.0003,
+      "loss": 0.34154040813446046,
+      "step": 840
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.3332885980606079,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.3266940116882324,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.306640625,
+      "learning_rate": 0.0003,
+      "loss": 0.32539618015289307,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.32114400863647463,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.296875,
+      "learning_rate": 0.0003,
+      "loss": 0.31635844707489014,
+      "step": 940
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.0003,
+      "loss": 0.31109464168548584,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.0003,
+      "loss": 0.30814523696899415,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.322265625,
+      "learning_rate": 0.0003,
+      "loss": 0.3065646648406982,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 0.3054979145526886,
+      "eval_runtime": 9.9252,
+      "eval_samples_per_second": 959.884,
+      "eval_steps_per_second": 15.012,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06875631951466127,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.3009540796279907,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07010448264239973,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.0003,
+      "loss": 0.29650895595550536,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07145264577013818,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.0003,
+      "loss": 0.2952280521392822,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07280080889787664,
+      "grad_norm": 0.30078125,
+      "learning_rate": 0.0003,
+      "loss": 0.2900059461593628,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "grad_norm": 1.359375,
+      "learning_rate": 0.0003,
+      "loss": 0.3038396596908569,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07549713515335356,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 0.3016719102859497,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07684529828109202,
+      "grad_norm": 0.326171875,
+      "learning_rate": 0.0003,
+      "loss": 0.2863359212875366,
+      "step": 1140
+    },
+    {
+      "epoch": 0.07819346140883048,
+      "grad_norm": 0.2734375,
+      "learning_rate": 0.0003,
+      "loss": 0.28296785354614257,
+      "step": 1160
+    },
+    {
+      "epoch": 0.07954162453656892,
+      "grad_norm": 0.75,
+      "learning_rate": 0.0003,
+      "loss": 0.2871352434158325,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.0003,
+      "loss": 0.2826373100280762,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "eval_loss": 0.28042274713516235,
+      "eval_runtime": 9.9396,
+      "eval_samples_per_second": 958.493,
+      "eval_steps_per_second": 14.991,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08223795079204584,
+      "grad_norm": 0.259765625,
+      "learning_rate": 0.0003,
+      "loss": 0.2763340473175049,
+      "step": 1220
+    },
+    {
+      "epoch": 0.0835861139197843,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 0.0003,
+      "loss": 0.2739908695220947,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08493427704752275,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 0.0003,
+      "loss": 0.27242236137390136,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08628244017526121,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 0.0003,
+      "loss": 0.2671719789505005,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "grad_norm": 0.2490234375,
+      "learning_rate": 0.0003,
+      "loss": 0.26690661907196045,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08897876643073811,
+      "grad_norm": 0.2421875,
+      "learning_rate": 0.0003,
+      "loss": 0.2639945030212402,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09032692955847657,
+      "grad_norm": 0.2421875,
+      "learning_rate": 0.0003,
+      "loss": 0.2623464584350586,
+      "step": 1340
+    },
+    {
+      "epoch": 0.09167509268621503,
+      "grad_norm": 0.244140625,
+      "learning_rate": 0.0003,
+      "loss": 0.25950236320495607,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 0.2412109375,
+      "learning_rate": 0.0003,
+      "loss": 0.2585299015045166,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "grad_norm": 0.236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.2554842233657837,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "eval_loss": 0.2570951581001282,
+      "eval_runtime": 10.0287,
+      "eval_samples_per_second": 949.974,
+      "eval_steps_per_second": 14.857,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0957195820694304,
+      "grad_norm": 0.2421875,
+      "learning_rate": 0.0003,
+      "loss": 0.25472869873046877,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09706774519716886,
+      "grad_norm": 0.23828125,
+      "learning_rate": 0.0003,
+      "loss": 0.2515384674072266,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09841590832490732,
+      "grad_norm": 0.236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.24910502433776854,
+      "step": 1460
+    },
+    {
+      "epoch": 0.09976407145264576,
+      "grad_norm": 0.228515625,
+      "learning_rate": 0.0003,
+      "loss": 0.2469315767288208,
+      "step": 1480
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "grad_norm": 0.236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.24581990242004395,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1016309219328000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

out/glu-situglu-21L_run/checkpoint-1500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711ad909475cdef5e2f24026d199875e77c559083ded851f0b6f7b4ae9dadd26
+size 4920

out/glu-situglu-21L_run/checkpoint-2000/config.json

@@ -0,0 +1,35 @@
+{
+  "activation": "situglu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 21,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096
+}

out/glu-situglu-21L_run/checkpoint-2000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89b7f1f86990c23530b4b8391a262e32b97a3bb1431c29840afa07c7a1d9e7ba
+size 7961656

out/glu-situglu-21L_run/checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb38cdaf1a520e20639060b5949a652dc37e7bbbe40192aa7527333b5762dc29
+size 16041274

out/glu-situglu-21L_run/checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a97db8e41139aa1239ba7fb79ddeb0af5998c6305a440c1fe182e6ad02f2f5
+size 14244

out/glu-situglu-21L_run/checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e91519ad666af6acc4beafd44aa617aefb90f632388a0f75cbf88f957400c190
+size 1064

out/glu-situglu-21L_run/checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

out/glu-situglu-21L_run/checkpoint-2000/trainer_state.json

@@ -0,0 +1,814 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.13481631277384565,
+  "eval_steps": 200,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.2578125,
+      "learning_rate": 0.0003,
+      "loss": 7.813759613037109,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 7.046382141113281,
+      "step": 40
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.98046875,
+      "learning_rate": 0.0003,
+      "loss": 6.4861198425292965,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.0003,
+      "loss": 6.114296722412109,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 2.375,
+      "learning_rate": 0.0003,
+      "loss": 5.7780506134033205,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.0003,
+      "loss": 5.281050491333008,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 2.953125,
+      "learning_rate": 0.0003,
+      "loss": 4.785240554809571,
+      "step": 140
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 2.921875,
+      "learning_rate": 0.0003,
+      "loss": 4.324177551269531,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 2.65625,
+      "learning_rate": 0.0003,
+      "loss": 3.9370296478271483,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 2.78125,
+      "learning_rate": 0.0003,
+      "loss": 3.5706451416015623,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 3.388801336288452,
+      "eval_runtime": 9.9378,
+      "eval_samples_per_second": 958.66,
+      "eval_steps_per_second": 14.993,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 3.21875,
+      "learning_rate": 0.0003,
+      "loss": 3.235015106201172,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 2.25,
+      "learning_rate": 0.0003,
+      "loss": 2.9087091445922852,
+      "step": 240
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 1.984375,
+      "learning_rate": 0.0003,
+      "loss": 2.606447219848633,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 2.28125,
+      "learning_rate": 0.0003,
+      "loss": 2.309796142578125,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 1.5859375,
+      "learning_rate": 0.0003,
+      "loss": 2.038427734375,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 1.65625,
+      "learning_rate": 0.0003,
+      "loss": 1.7724691390991212,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 1.6171875,
+      "learning_rate": 0.0003,
+      "loss": 1.5364046096801758,
+      "step": 340
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.0003,
+      "loss": 1.3601617813110352,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 1.203125,
+      "learning_rate": 0.0003,
+      "loss": 1.1919203758239747,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.97265625,
+      "learning_rate": 0.0003,
+      "loss": 1.031599998474121,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 0.9726812839508057,
+      "eval_runtime": 9.9094,
+      "eval_samples_per_second": 961.411,
+      "eval_steps_per_second": 15.036,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 1.078125,
+      "learning_rate": 0.0003,
+      "loss": 0.9072963714599609,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 1.0859375,
+      "learning_rate": 0.0003,
+      "loss": 0.8252955436706543,
+      "step": 440
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0003,
+      "loss": 0.7282839775085449,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.48828125,
+      "learning_rate": 0.0003,
+      "loss": 0.6567337512969971,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.380859375,
+      "learning_rate": 0.0003,
+      "loss": 0.5960001945495605,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.359375,
+      "learning_rate": 0.0003,
+      "loss": 0.5524660587310791,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.0003,
+      "loss": 0.5125324249267578,
+      "step": 540
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.37109375,
+      "learning_rate": 0.0003,
+      "loss": 0.4881768226623535,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.35546875,
+      "learning_rate": 0.0003,
+      "loss": 0.463836669921875,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003,
+      "loss": 0.44971308708190916,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 0.4426644444465637,
+      "eval_runtime": 9.9173,
+      "eval_samples_per_second": 960.644,
+      "eval_steps_per_second": 15.024,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.365234375,
+      "learning_rate": 0.0003,
+      "loss": 0.4324655055999756,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.392578125,
+      "learning_rate": 0.0003,
+      "loss": 0.4214168548583984,
+      "step": 640
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.0003,
+      "loss": 0.4324348449707031,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.48046875,
+      "learning_rate": 0.0003,
+      "loss": 0.4121569633483887,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.3203125,
+      "learning_rate": 0.0003,
+      "loss": 0.3938265800476074,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.28515625,
+      "learning_rate": 0.0003,
+      "loss": 0.3844801664352417,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.3729189395904541,
+      "step": 740
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.28125,
+      "learning_rate": 0.0003,
+      "loss": 0.3688415765762329,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.36176743507385256,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.0003,
+      "loss": 0.3517471790313721,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 0.35235798358917236,
+      "eval_runtime": 10.0631,
+      "eval_samples_per_second": 946.728,
+      "eval_steps_per_second": 14.807,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.34776208400726316,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.0003,
+      "loss": 0.34154040813446046,
+      "step": 840
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.3332885980606079,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.279296875,
+      "learning_rate": 0.0003,
+      "loss": 0.3266940116882324,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.306640625,
+      "learning_rate": 0.0003,
+      "loss": 0.32539618015289307,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.32114400863647463,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.296875,
+      "learning_rate": 0.0003,
+      "loss": 0.31635844707489014,
+      "step": 940
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.0003,
+      "loss": 0.31109464168548584,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.0003,
+      "loss": 0.30814523696899415,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.322265625,
+      "learning_rate": 0.0003,
+      "loss": 0.3065646648406982,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 0.3054979145526886,
+      "eval_runtime": 9.9252,
+      "eval_samples_per_second": 959.884,
+      "eval_steps_per_second": 15.012,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06875631951466127,
+      "grad_norm": 0.294921875,
+      "learning_rate": 0.0003,
+      "loss": 0.3009540796279907,
+      "step": 1020
+    },
+    {
+      "epoch": 0.07010448264239973,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.0003,
+      "loss": 0.29650895595550536,
+      "step": 1040
+    },
+    {
+      "epoch": 0.07145264577013818,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.0003,
+      "loss": 0.2952280521392822,
+      "step": 1060
+    },
+    {
+      "epoch": 0.07280080889787664,
+      "grad_norm": 0.30078125,
+      "learning_rate": 0.0003,
+      "loss": 0.2900059461593628,
+      "step": 1080
+    },
+    {
+      "epoch": 0.0741489720256151,
+      "grad_norm": 1.359375,
+      "learning_rate": 0.0003,
+      "loss": 0.3038396596908569,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07549713515335356,
+      "grad_norm": 1.1796875,
+      "learning_rate": 0.0003,
+      "loss": 0.3016719102859497,
+      "step": 1120
+    },
+    {
+      "epoch": 0.07684529828109202,
+      "grad_norm": 0.326171875,
+      "learning_rate": 0.0003,
+      "loss": 0.2863359212875366,
+      "step": 1140
+    },
+    {
+      "epoch": 0.07819346140883048,
+      "grad_norm": 0.2734375,
+      "learning_rate": 0.0003,
+      "loss": 0.28296785354614257,
+      "step": 1160
+    },
+    {
+      "epoch": 0.07954162453656892,
+      "grad_norm": 0.75,
+      "learning_rate": 0.0003,
+      "loss": 0.2871352434158325,
+      "step": 1180
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.0003,
+      "loss": 0.2826373100280762,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08088978766430738,
+      "eval_loss": 0.28042274713516235,
+      "eval_runtime": 9.9396,
+      "eval_samples_per_second": 958.493,
+      "eval_steps_per_second": 14.991,
+      "step": 1200
+    },
+    {
+      "epoch": 0.08223795079204584,
+      "grad_norm": 0.259765625,
+      "learning_rate": 0.0003,
+      "loss": 0.2763340473175049,
+      "step": 1220
+    },
+    {
+      "epoch": 0.0835861139197843,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 0.0003,
+      "loss": 0.2739908695220947,
+      "step": 1240
+    },
+    {
+      "epoch": 0.08493427704752275,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 0.0003,
+      "loss": 0.27242236137390136,
+      "step": 1260
+    },
+    {
+      "epoch": 0.08628244017526121,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 0.0003,
+      "loss": 0.2671719789505005,
+      "step": 1280
+    },
+    {
+      "epoch": 0.08763060330299967,
+      "grad_norm": 0.2490234375,
+      "learning_rate": 0.0003,
+      "loss": 0.26690661907196045,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08897876643073811,
+      "grad_norm": 0.2421875,
+      "learning_rate": 0.0003,
+      "loss": 0.2639945030212402,
+      "step": 1320
+    },
+    {
+      "epoch": 0.09032692955847657,
+      "grad_norm": 0.2421875,
+      "learning_rate": 0.0003,
+      "loss": 0.2623464584350586,
+      "step": 1340
+    },
+    {
+      "epoch": 0.09167509268621503,
+      "grad_norm": 0.244140625,
+      "learning_rate": 0.0003,
+      "loss": 0.25950236320495607,
+      "step": 1360
+    },
+    {
+      "epoch": 0.09302325581395349,
+      "grad_norm": 0.2412109375,
+      "learning_rate": 0.0003,
+      "loss": 0.2585299015045166,
+      "step": 1380
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "grad_norm": 0.236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.2554842233657837,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09437141894169195,
+      "eval_loss": 0.2570951581001282,
+      "eval_runtime": 10.0287,
+      "eval_samples_per_second": 949.974,
+      "eval_steps_per_second": 14.857,
+      "step": 1400
+    },
+    {
+      "epoch": 0.0957195820694304,
+      "grad_norm": 0.2421875,
+      "learning_rate": 0.0003,
+      "loss": 0.25472869873046877,
+      "step": 1420
+    },
+    {
+      "epoch": 0.09706774519716886,
+      "grad_norm": 0.23828125,
+      "learning_rate": 0.0003,
+      "loss": 0.2515384674072266,
+      "step": 1440
+    },
+    {
+      "epoch": 0.09841590832490732,
+      "grad_norm": 0.236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.24910502433776854,
+      "step": 1460
+    },
+    {
+      "epoch": 0.09976407145264576,
+      "grad_norm": 0.228515625,
+      "learning_rate": 0.0003,
+      "loss": 0.2469315767288208,
+      "step": 1480
+    },
+    {
+      "epoch": 0.10111223458038422,
+      "grad_norm": 0.236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.24581990242004395,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10246039770812268,
+      "grad_norm": 0.2392578125,
+      "learning_rate": 0.0003,
+      "loss": 0.24391622543334962,
+      "step": 1520
+    },
+    {
+      "epoch": 0.10380856083586114,
+      "grad_norm": 0.228515625,
+      "learning_rate": 0.0003,
+      "loss": 0.243127179145813,
+      "step": 1540
+    },
+    {
+      "epoch": 0.1051567239635996,
+      "grad_norm": 0.234375,
+      "learning_rate": 0.0003,
+      "loss": 0.24260540008544923,
+      "step": 1560
+    },
+    {
+      "epoch": 0.10650488709133805,
+      "grad_norm": 0.2373046875,
+      "learning_rate": 0.0003,
+      "loss": 0.24284977912902833,
+      "step": 1580
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "grad_norm": 1.140625,
+      "learning_rate": 0.0003,
+      "loss": 0.27417311668395994,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10785305021907651,
+      "eval_loss": 0.2722279131412506,
+      "eval_runtime": 9.9812,
+      "eval_samples_per_second": 954.493,
+      "eval_steps_per_second": 14.928,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10920121334681497,
+      "grad_norm": 0.318359375,
+      "learning_rate": 0.0003,
+      "loss": 0.25417091846466067,
+      "step": 1620
+    },
+    {
+      "epoch": 0.11054937647455342,
+      "grad_norm": 0.2314453125,
+      "learning_rate": 0.0003,
+      "loss": 0.24314517974853517,
+      "step": 1640
+    },
+    {
+      "epoch": 0.11189753960229187,
+      "grad_norm": 0.2265625,
+      "learning_rate": 0.0003,
+      "loss": 0.23854396343231202,
+      "step": 1660
+    },
+    {
+      "epoch": 0.11324570273003033,
+      "grad_norm": 0.224609375,
+      "learning_rate": 0.0003,
+      "loss": 0.23716413974761963,
+      "step": 1680
+    },
+    {
+      "epoch": 0.11459386585776879,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.23724014759063722,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11594202898550725,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 0.0003,
+      "loss": 0.23469991683959962,
+      "step": 1720
+    },
+    {
+      "epoch": 0.1172901921132457,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 0.0003,
+      "loss": 0.23476831912994384,
+      "step": 1740
+    },
+    {
+      "epoch": 0.11863835524098416,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 0.0003,
+      "loss": 0.23299741744995117,
+      "step": 1760
+    },
+    {
+      "epoch": 0.11998651836872262,
+      "grad_norm": 0.216796875,
+      "learning_rate": 0.0003,
+      "loss": 0.23383264541625975,
+      "step": 1780
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "grad_norm": 0.2197265625,
+      "learning_rate": 0.0003,
+      "loss": 0.2340226173400879,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12133468149646107,
+      "eval_loss": 0.23292653262615204,
+      "eval_runtime": 10.0275,
+      "eval_samples_per_second": 950.091,
+      "eval_steps_per_second": 14.859,
+      "step": 1800
+    },
+    {
+      "epoch": 0.12268284462419952,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.2334127187728882,
+      "step": 1820
+    },
+    {
+      "epoch": 0.12403100775193798,
+      "grad_norm": 0.21875,
+      "learning_rate": 0.0003,
+      "loss": 0.22844159603118896,
+      "step": 1840
+    },
+    {
+      "epoch": 0.12537917087967643,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 0.0003,
+      "loss": 0.22850356101989747,
+      "step": 1860
+    },
+    {
+      "epoch": 0.1267273340074149,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 0.0003,
+      "loss": 0.2282630205154419,
+      "step": 1880
+    },
+    {
+      "epoch": 0.12807549713515334,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 0.0003,
+      "loss": 0.22788019180297853,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12942366026289182,
+      "grad_norm": 0.21875,
+      "learning_rate": 0.0003,
+      "loss": 0.22555692195892335,
+      "step": 1920
+    },
+    {
+      "epoch": 0.13077182339063026,
+      "grad_norm": 0.21875,
+      "learning_rate": 0.0003,
+      "loss": 0.2264007329940796,
+      "step": 1940
+    },
+    {
+      "epoch": 0.13211998651836873,
+      "grad_norm": 0.216796875,
+      "learning_rate": 0.0003,
+      "loss": 0.2237532138824463,
+      "step": 1960
+    },
+    {
+      "epoch": 0.13346814964610718,
+      "grad_norm": 0.220703125,
+      "learning_rate": 0.0003,
+      "loss": 0.2249152898788452,
+      "step": 1980
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 0.0003,
+      "loss": 0.22449655532836915,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13481631277384565,
+      "eval_loss": 0.22595776617527008,
+      "eval_runtime": 10.0761,
+      "eval_samples_per_second": 945.505,
+      "eval_steps_per_second": 14.787,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1355078959104000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

out/glu-situglu-21L_run/checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711ad909475cdef5e2f24026d199875e77c559083ded851f0b6f7b4ae9dadd26
+size 4920

out/glu-situglu-21L_run/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b38378164539aa0dc545450c2631ed5ba25590e597de010606c3bc5f2d0aaed
+oid sha256:89b7f1f86990c23530b4b8391a262e32b97a3bb1431c29840afa07c7a1d9e7ba
 size 7961656

out/glu-situglu-21L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

out/sweep_summary.json

@@ -1,8 +1,8 @@
 [
   {
-    "variant": "glu-linear-21L",
-    "eval_loss": 0.21702784299850464,
-    "out": "out/glu-linear-21L_run",
-    "run_name": "LM-glu-linear-21L-4.0M-20260806-211222"
+    "variant": "glu-situglu-21L",
+    "eval_loss": 0.22595776617527008,
+    "out": "out/glu-situglu-21L_run",
+    "run_name": "LM-glu-situglu-21L-4.0M-20260806-211757"
   }
 ]

wandb/debug-internal.log

@@ -1,31 +1,25 @@
-{"time":"2026-08-06T21:17:58.671634953Z","level":"INFO","msg":"wandb-core"}
-{"time":"2026-08-06T21:17:58.671778326Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
-{"time":"2026-08-06T21:17:58.929335556Z","level":"INFO","msg":"stream: created new stream","id":"et66wpfp"}
-{"time":"2026-08-06T21:17:58.929407279Z","level":"INFO","msg":"handler: started"}
-{"time":"2026-08-06T21:17:58.929495955Z","level":"INFO","msg":"stream: started"}
-{"time":"2026-08-06T21:17:58.929509199Z","level":"INFO","msg":"writer: started","stream_id":"et66wpfp"}
-{"time":"2026-08-06T21:17:58.929535303Z","level":"INFO","msg":"sender: started"}
-{"time":"2026-08-06T21:17:59.433621649Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
-{"time":"2026-08-06T21:17:59.520631931Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:18:02.550035484Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":75}
-{"time":"2026-08-06T21:18:02.586349041Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":4342}
-{"time":"2026-08-06T21:18:14.443726541Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":6,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":10,"uploaded_len":2}
-{"time":"2026-08-06T21:18:14.790659274Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:18:29.440596176Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":4,"events_offset":2,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-06T21:18:29.773980786Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:18:44.444963357Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":6,"events_offset":4,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-06T21:18:44.822078717Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:18:59.437526221Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":5,"events_offset":6,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-06T21:18:59.782923067Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:19:14.441917627Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":4,"events_offset":8,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-06T21:19:14.789677084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:19:29.437648423Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":7,"events_offset":10,"events_lines":2,"console_offset":3,"console_lines":1}
-{"time":"2026-08-06T21:19:29.819436354Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:19:44.442070317Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":3,"events_offset":12,"events_lines":2,"console_offset":10,"console_lines":36}
-{"time":"2026-08-06T21:19:44.79296215Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:19:59.437465828Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":7,"events_offset":14,"events_lines":2,"console_offset":43,"console_lines":1}
-{"time":"2026-08-06T21:19:59.702421474Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:20:14.441573932Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":42,"history_lines":3,"events_offset":16,"events_lines":2,"console_offset":43,"console_lines":1}
-{"time":"2026-08-06T21:20:14.763680337Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-06T21:20:29.438442012Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":45,"history_lines":7,"events_offset":18,"events_lines":2,"console_offset":43,"console_lines":1}
-{"time":"2026-08-06T21:20:29.75683591Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:23:49.262455471Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-06T21:23:49.262578674Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-06T21:23:49.520065686Z","level":"INFO","msg":"stream: created new stream","id":"khl56ofx"}
+{"time":"2026-08-06T21:23:49.520128136Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-06T21:23:49.520211964Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-06T21:23:49.520222301Z","level":"INFO","msg":"writer: started","stream_id":"khl56ofx"}
+{"time":"2026-08-06T21:23:49.520242836Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-06T21:23:50.032944406Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-06T21:23:50.120147106Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:23:53.039944789Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":75}
+{"time":"2026-08-06T21:23:53.076530748Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":4216}
+{"time":"2026-08-06T21:24:05.04332536Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":6,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":9,"uploaded_len":2}
+{"time":"2026-08-06T21:24:05.435244467Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:24:20.039026109Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":4,"events_offset":2,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-06T21:24:20.389487595Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:24:35.040700155Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":6,"events_offset":4,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-06T21:24:35.368304724Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:24:50.038230763Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":5,"events_offset":6,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-06T21:24:50.31147386Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:25:05.040055585Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":5,"events_offset":8,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-06T21:25:05.349758223Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:25:20.037151722Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":6,"events_offset":10,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-06T21:25:20.358886366Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-06T21:25:35.040574125Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":4,"events_offset":12,"events_lines":2,"console_offset":10,"console_lines":37}
+{"time":"2026-08-06T21:25:35.362028876Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

wandb/debug.log

@@ -1,23 +1,23 @@
-2026-08-06 21:17:58,669 INFO    MainThread:867849 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_setup.py:_flush():81] Configure stats pid to 867849
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/Activation/wandb/run-20260806_211758-et66wpfp/logs/debug.log
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/Activation/wandb/run-20260806_211758-et66wpfp/logs/debug-internal.log
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_init.py:init():772] calling init triggers
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+2026-08-06 21:23:49,260 INFO    MainThread:922400 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_setup.py:_flush():81] Configure stats pid to 922400
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/Activation/wandb/run-20260806_212349-khl56ofx/logs/debug.log
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/Activation/wandb/run-20260806_212349-khl56ofx/logs/debug-internal.log
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_init.py:init():772] calling init triggers
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_init.py:init():820] starting backend
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
-2026-08-06 21:17:58,670 INFO    MainThread:867849 [wandb_init.py:init():835] sending inform_init request
-2026-08-06 21:17:58,929 INFO    MainThread:867849 [wandb_init.py:init():840] backend started and connected
-2026-08-06 21:17:58,933 INFO    MainThread:867849 [wandb_init.py:init():910] updated telemetry
-2026-08-06 21:17:58,940 INFO    MainThread:867849 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
-2026-08-06 21:17:59,169 INFO    MainThread:867849 [wandb_init.py:init():978] starting run threads in backend
-2026-08-06 21:17:59,242 INFO    MainThread:867849 [wandb_run.py:_console_start():2621] atexit reg
-2026-08-06 21:17:59,243 INFO    MainThread:867849 [wandb_run.py:_redirect():2471] redirect: wrap_raw
-2026-08-06 21:17:59,243 INFO    MainThread:867849 [wandb_run.py:_redirect():2540] Wrapping output streams.
-2026-08-06 21:17:59,243 INFO    MainThread:867849 [wandb_run.py:_redirect():2563] Redirects installed.
-2026-08-06 21:17:59,246 INFO    MainThread:867849 [wandb_init.py:init():1016] run started, returning control to user process
-2026-08-06 21:17:59,246 INFO    MainThread:867849 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 21, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'situglu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-situglu-21L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 2000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-situglu-21L-4.0M-20260806-211757', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 200, 'eval_delay': 0, 'per_device_eval_batch_size': 64, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 500, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': True, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/ACTio-glu-situglu-21L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
-2026-08-06 21:17:59,249 INFO    MainThread:867849 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 3970432 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x151e0c293690>>
-2026-08-06 21:17:59,249 INFO    MainThread:867849 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 3970432 None
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_init.py:init():820] starting backend
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-06 21:23:49,261 INFO    MainThread:922400 [wandb_init.py:init():835] sending inform_init request
+2026-08-06 21:23:49,520 INFO    MainThread:922400 [wandb_init.py:init():840] backend started and connected
+2026-08-06 21:23:49,523 INFO    MainThread:922400 [wandb_init.py:init():910] updated telemetry
+2026-08-06 21:23:49,530 INFO    MainThread:922400 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-06 21:23:49,772 INFO    MainThread:922400 [wandb_init.py:init():978] starting run threads in backend
+2026-08-06 21:23:49,845 INFO    MainThread:922400 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-06 21:23:49,845 INFO    MainThread:922400 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-06 21:23:49,845 INFO    MainThread:922400 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-06 21:23:49,845 INFO    MainThread:922400 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-06 21:23:49,848 INFO    MainThread:922400 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-06 21:23:49,849 INFO    MainThread:922400 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 21, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 's10', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-s10-21L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 2000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-s10-21L-4.0M-20260806-212348', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 200, 'eval_delay': 0, 'per_device_eval_batch_size': 64, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 500, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': True, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/ACTio-glu-s10-21L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-06 21:23:49,851 INFO    MainThread:922400 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 3970432 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x154115817dd0>>
+2026-08-06 21:23:49,851 INFO    MainThread:922400 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 3970432 None