Auto upload 2026-08-03T21:05:22.526000

out/gelu_run/README.md

@@ -0,0 +1,77 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+model-index:
+- name: tiny-llama-3-gelu
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# tiny-llama-3-gelu
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.3376
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 32
+- eval_batch_size: 32
+- seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: constant
+- training_steps: 10000
+
+### Training results
+
+| Training Loss | Epoch  | Step  | Validation Loss |
+|:-------------:|:------:|:-----:|:---------------:|
+| 4.7181        | 0.0319 | 500   | 2.3297          |
+| 3.8008        | 0.0637 | 1000  | 1.9001          |
+| 3.4600        | 0.0956 | 1500  | 1.7322          |
+| 3.2888        | 0.1274 | 2000  | 1.6443          |
+| 3.1567        | 0.1593 | 2500  | 1.5846          |
+| 3.0631        | 0.1911 | 3000  | 1.5423          |
+| 3.0362        | 0.2230 | 3500  | 1.5104          |
+| 2.9560        | 0.2548 | 4000  | 1.4824          |
+| 2.9084        | 0.2867 | 4500  | 1.4600          |
+| 2.8723        | 0.3185 | 5000  | 1.4424          |
+| 2.8515        | 0.3504 | 5500  | 1.4256          |
+| 2.8194        | 0.3823 | 6000  | 1.4123          |
+| 2.7946        | 0.4141 | 6500  | 1.3995          |
+| 2.7780        | 0.4460 | 7000  | 1.3887          |
+| 2.7546        | 0.4778 | 7500  | 1.3774          |
+| 2.7302        | 0.5097 | 8000  | 1.3686          |
+| 2.7196        | 0.5415 | 8500  | 1.3599          |
+| 2.7139        | 0.5734 | 9000  | 1.3530          |
+| 2.6833        | 0.6052 | 9500  | 1.3448          |
+| 2.6637        | 0.6371 | 10000 | 1.3376          |
+
+
+### Framework versions
+
+- Transformers 5.15.0.dev0
+- Pytorch 2.6.0+cu124
+- Datasets 5.0.1
+- Tokenizers 0.22.2

out/gelu_run/chat_template.jinja

@@ -0,0 +1,15 @@
+{% for message in messages %}
+{% if message['role'] == 'user' %}
+{{ '<|user|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'system' %}
+{{ '<|system|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'assistant' %}
+{{ '<|assistant|>
+'  + message['content'] + eos_token }}
+{% endif %}
+{% if loop.last and add_generation_prompt %}
+{{ '<|assistant|>' }}
+{% endif %}
+{% endfor %}

out/gelu_run/checkpoint-10000/chat_template.jinja

@@ -0,0 +1,15 @@
+{% for message in messages %}
+{% if message['role'] == 'user' %}
+{{ '<|user|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'system' %}
+{{ '<|system|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'assistant' %}
+{{ '<|assistant|>
+'  + message['content'] + eos_token }}
+{% endif %}
+{% if loop.last and add_generation_prompt %}
+{{ '<|assistant|>' }}
+{% endif %}
+{% endfor %}

out/gelu_run/checkpoint-10000/config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "glu_activation": "gelu",
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "tiny_llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

out/gelu_run/checkpoint-10000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db6e127711dc7a082314a726aa5990adf7c2d5d2be930edd888d5efe90fa2260
+size 208226064

out/gelu_run/checkpoint-10000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef72d5c25f4a96a7e8b5666e4bd32ff3a4fe20e7894b3420bc4567edd463e981
+size 416543034

out/gelu_run/checkpoint-10000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
+size 14244

out/gelu_run/checkpoint-10000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4943a4b68e3bb4800466ee6c597a8242d165d5e3adaf59ebce14536925495437
+size 1064

out/gelu_run/checkpoint-10000/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

out/gelu_run/checkpoint-10000/trainer_state.json

@@ -0,0 +1,1594 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6370847004109196,
+  "eval_steps": 500,
+  "global_step": 10000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003185423502054598,
+      "grad_norm": 5.359443187713623,
+      "learning_rate": 0.0003,
+      "loss": 13.56220703125,
+      "step": 50
+    },
+    {
+      "epoch": 0.006370847004109196,
+      "grad_norm": 6.136266708374023,
+      "learning_rate": 0.0003,
+      "loss": 10.229112548828125,
+      "step": 100
+    },
+    {
+      "epoch": 0.009556270506163795,
+      "grad_norm": 2.364473342895508,
+      "learning_rate": 0.0003,
+      "loss": 8.330999145507812,
+      "step": 150
+    },
+    {
+      "epoch": 0.012741694008218393,
+      "grad_norm": 2.0175695419311523,
+      "learning_rate": 0.0003,
+      "loss": 7.029268798828125,
+      "step": 200
+    },
+    {
+      "epoch": 0.01592711751027299,
+      "grad_norm": 1.450079321861267,
+      "learning_rate": 0.0003,
+      "loss": 6.264676513671875,
+      "step": 250
+    },
+    {
+      "epoch": 0.01911254101232759,
+      "grad_norm": 1.9761725664138794,
+      "learning_rate": 0.0003,
+      "loss": 5.7695263671875,
+      "step": 300
+    },
+    {
+      "epoch": 0.022297964514382186,
+      "grad_norm": 1.4381787776947021,
+      "learning_rate": 0.0003,
+      "loss": 5.411314086914063,
+      "step": 350
+    },
+    {
+      "epoch": 0.025483388016436785,
+      "grad_norm": 1.321711778640747,
+      "learning_rate": 0.0003,
+      "loss": 5.134971923828125,
+      "step": 400
+    },
+    {
+      "epoch": 0.028668811518491384,
+      "grad_norm": 1.2103067636489868,
+      "learning_rate": 0.0003,
+      "loss": 4.9218051147460935,
+      "step": 450
+    },
+    {
+      "epoch": 0.03185423502054598,
+      "grad_norm": 1.212465763092041,
+      "learning_rate": 0.0003,
+      "loss": 4.718062133789062,
+      "step": 500
+    },
+    {
+      "epoch": 0.03185423502054598,
+      "eval_loss": 2.329653739929199,
+      "eval_runtime": 23.219,
+      "eval_samples_per_second": 435.117,
+      "eval_steps_per_second": 13.61,
+      "step": 500
+    },
+    {
+      "epoch": 0.03503965852260058,
+      "grad_norm": 1.1227043867111206,
+      "learning_rate": 0.0003,
+      "loss": 4.5997119140625,
+      "step": 550
+    },
+    {
+      "epoch": 0.03822508202465518,
+      "grad_norm": 1.1499993801116943,
+      "learning_rate": 0.0003,
+      "loss": 4.4310519409179685,
+      "step": 600
+    },
+    {
+      "epoch": 0.04141050552670977,
+      "grad_norm": 1.0175905227661133,
+      "learning_rate": 0.0003,
+      "loss": 4.341097106933594,
+      "step": 650
+    },
+    {
+      "epoch": 0.04459592902876437,
+      "grad_norm": 1.0321624279022217,
+      "learning_rate": 0.0003,
+      "loss": 4.231830139160156,
+      "step": 700
+    },
+    {
+      "epoch": 0.04778135253081897,
+      "grad_norm": 1.0309418439865112,
+      "learning_rate": 0.0003,
+      "loss": 4.140238952636719,
+      "step": 750
+    },
+    {
+      "epoch": 0.05096677603287357,
+      "grad_norm": 0.940864086151123,
+      "learning_rate": 0.0003,
+      "loss": 4.054711608886719,
+      "step": 800
+    },
+    {
+      "epoch": 0.05415219953492817,
+      "grad_norm": 0.9923902153968811,
+      "learning_rate": 0.0003,
+      "loss": 3.980743713378906,
+      "step": 850
+    },
+    {
+      "epoch": 0.05733762303698277,
+      "grad_norm": 0.9455788731575012,
+      "learning_rate": 0.0003,
+      "loss": 3.914459228515625,
+      "step": 900
+    },
+    {
+      "epoch": 0.06052304653903737,
+      "grad_norm": 0.9779245257377625,
+      "learning_rate": 0.0003,
+      "loss": 3.8528009033203126,
+      "step": 950
+    },
+    {
+      "epoch": 0.06370847004109197,
+      "grad_norm": 0.9273849725723267,
+      "learning_rate": 0.0003,
+      "loss": 3.8008270263671875,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06370847004109197,
+      "eval_loss": 1.900104284286499,
+      "eval_runtime": 22.787,
+      "eval_samples_per_second": 443.367,
+      "eval_steps_per_second": 13.868,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06689389354314657,
+      "grad_norm": 0.9474796056747437,
+      "learning_rate": 0.0003,
+      "loss": 3.756812744140625,
+      "step": 1050
+    },
+    {
+      "epoch": 0.07007931704520116,
+      "grad_norm": 0.898605227470398,
+      "learning_rate": 0.0003,
+      "loss": 3.725189208984375,
+      "step": 1100
+    },
+    {
+      "epoch": 0.07326474054725576,
+      "grad_norm": 0.8247606754302979,
+      "learning_rate": 0.0003,
+      "loss": 3.69662109375,
+      "step": 1150
+    },
+    {
+      "epoch": 0.07645016404931036,
+      "grad_norm": 0.9039903879165649,
+      "learning_rate": 0.0003,
+      "loss": 3.633312072753906,
+      "step": 1200
+    },
+    {
+      "epoch": 0.07963558755136495,
+      "grad_norm": 0.8472636342048645,
+      "learning_rate": 0.0003,
+      "loss": 3.627657470703125,
+      "step": 1250
+    },
+    {
+      "epoch": 0.08282101105341955,
+      "grad_norm": 0.8446575999259949,
+      "learning_rate": 0.0003,
+      "loss": 3.59127685546875,
+      "step": 1300
+    },
+    {
+      "epoch": 0.08600643455547415,
+      "grad_norm": 0.8907977342605591,
+      "learning_rate": 0.0003,
+      "loss": 3.557159423828125,
+      "step": 1350
+    },
+    {
+      "epoch": 0.08919185805752874,
+      "grad_norm": 0.8718058466911316,
+      "learning_rate": 0.0003,
+      "loss": 3.50177490234375,
+      "step": 1400
+    },
+    {
+      "epoch": 0.09237728155958334,
+      "grad_norm": 0.8583778738975525,
+      "learning_rate": 0.0003,
+      "loss": 3.522080078125,
+      "step": 1450
+    },
+    {
+      "epoch": 0.09556270506163794,
+      "grad_norm": 0.8243407607078552,
+      "learning_rate": 0.0003,
+      "loss": 3.4600424194335937,
+      "step": 1500
+    },
+    {
+      "epoch": 0.09556270506163794,
+      "eval_loss": 1.7322442531585693,
+      "eval_runtime": 22.9403,
+      "eval_samples_per_second": 440.405,
+      "eval_steps_per_second": 13.775,
+      "step": 1500
+    },
+    {
+      "epoch": 0.09874812856369254,
+      "grad_norm": 0.8302342295646667,
+      "learning_rate": 0.0003,
+      "loss": 3.4528384399414063,
+      "step": 1550
+    },
+    {
+      "epoch": 0.10193355206574714,
+      "grad_norm": 0.8258938193321228,
+      "learning_rate": 0.0003,
+      "loss": 3.4215289306640626,
+      "step": 1600
+    },
+    {
+      "epoch": 0.10511897556780174,
+      "grad_norm": 0.8112037181854248,
+      "learning_rate": 0.0003,
+      "loss": 3.4030706787109377,
+      "step": 1650
+    },
+    {
+      "epoch": 0.10830439906985634,
+      "grad_norm": 0.8116158246994019,
+      "learning_rate": 0.0003,
+      "loss": 3.39681884765625,
+      "step": 1700
+    },
+    {
+      "epoch": 0.11148982257191094,
+      "grad_norm": 0.8268960118293762,
+      "learning_rate": 0.0003,
+      "loss": 3.3635791015625,
+      "step": 1750
+    },
+    {
+      "epoch": 0.11467524607396554,
+      "grad_norm": 0.8487680554389954,
+      "learning_rate": 0.0003,
+      "loss": 3.3529998779296877,
+      "step": 1800
+    },
+    {
+      "epoch": 0.11786066957602014,
+      "grad_norm": 0.7939220666885376,
+      "learning_rate": 0.0003,
+      "loss": 3.342622985839844,
+      "step": 1850
+    },
+    {
+      "epoch": 0.12104609307807473,
+      "grad_norm": 0.7922126054763794,
+      "learning_rate": 0.0003,
+      "loss": 3.329544677734375,
+      "step": 1900
+    },
+    {
+      "epoch": 0.12423151658012933,
+      "grad_norm": 0.7864624261856079,
+      "learning_rate": 0.0003,
+      "loss": 3.3191567993164064,
+      "step": 1950
+    },
+    {
+      "epoch": 0.12741694008218393,
+      "grad_norm": 0.7754216194152832,
+      "learning_rate": 0.0003,
+      "loss": 3.288849182128906,
+      "step": 2000
+    },
+    {
+      "epoch": 0.12741694008218393,
+      "eval_loss": 1.6442697048187256,
+      "eval_runtime": 23.3516,
+      "eval_samples_per_second": 432.647,
+      "eval_steps_per_second": 13.532,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13060236358423852,
+      "grad_norm": 0.8032373785972595,
+      "learning_rate": 0.0003,
+      "loss": 3.2620733642578124,
+      "step": 2050
+    },
+    {
+      "epoch": 0.13378778708629313,
+      "grad_norm": 0.7730779647827148,
+      "learning_rate": 0.0003,
+      "loss": 3.2765151977539064,
+      "step": 2100
+    },
+    {
+      "epoch": 0.13697321058834772,
+      "grad_norm": 0.804121196269989,
+      "learning_rate": 0.0003,
+      "loss": 3.2524420166015626,
+      "step": 2150
+    },
+    {
+      "epoch": 0.14015863409040233,
+      "grad_norm": 0.8174483180046082,
+      "learning_rate": 0.0003,
+      "loss": 3.2468771362304687,
+      "step": 2200
+    },
+    {
+      "epoch": 0.1433440575924569,
+      "grad_norm": 0.775197446346283,
+      "learning_rate": 0.0003,
+      "loss": 3.22389404296875,
+      "step": 2250
+    },
+    {
+      "epoch": 0.14652948109451153,
+      "grad_norm": 0.754429280757904,
+      "learning_rate": 0.0003,
+      "loss": 3.213176574707031,
+      "step": 2300
+    },
+    {
+      "epoch": 0.1497149045965661,
+      "grad_norm": 0.7735773324966431,
+      "learning_rate": 0.0003,
+      "loss": 3.2035504150390626,
+      "step": 2350
+    },
+    {
+      "epoch": 0.15290032809862072,
+      "grad_norm": 0.7535238862037659,
+      "learning_rate": 0.0003,
+      "loss": 3.186253967285156,
+      "step": 2400
+    },
+    {
+      "epoch": 0.1560857516006753,
+      "grad_norm": 0.776179850101471,
+      "learning_rate": 0.0003,
+      "loss": 3.17057861328125,
+      "step": 2450
+    },
+    {
+      "epoch": 0.1592711751027299,
+      "grad_norm": 0.7464358806610107,
+      "learning_rate": 0.0003,
+      "loss": 3.156702880859375,
+      "step": 2500
+    },
+    {
+      "epoch": 0.1592711751027299,
+      "eval_loss": 1.5845896005630493,
+      "eval_runtime": 22.1836,
+      "eval_samples_per_second": 455.427,
+      "eval_steps_per_second": 14.245,
+      "step": 2500
+    },
+    {
+      "epoch": 0.1624565986047845,
+      "grad_norm": 0.7600530385971069,
+      "learning_rate": 0.0003,
+      "loss": 3.151944885253906,
+      "step": 2550
+    },
+    {
+      "epoch": 0.1656420221068391,
+      "grad_norm": 0.7889643907546997,
+      "learning_rate": 0.0003,
+      "loss": 3.136610107421875,
+      "step": 2600
+    },
+    {
+      "epoch": 0.1688274456088937,
+      "grad_norm": 0.7493102550506592,
+      "learning_rate": 0.0003,
+      "loss": 3.1313116455078127,
+      "step": 2650
+    },
+    {
+      "epoch": 0.1720128691109483,
+      "grad_norm": 0.7583590745925903,
+      "learning_rate": 0.0003,
+      "loss": 3.1122531127929687,
+      "step": 2700
+    },
+    {
+      "epoch": 0.1751982926130029,
+      "grad_norm": 0.7713261246681213,
+      "learning_rate": 0.0003,
+      "loss": 3.1124774169921876,
+      "step": 2750
+    },
+    {
+      "epoch": 0.1783837161150575,
+      "grad_norm": 0.7436685562133789,
+      "learning_rate": 0.0003,
+      "loss": 3.1192745971679687,
+      "step": 2800
+    },
+    {
+      "epoch": 0.1815691396171121,
+      "grad_norm": 0.7387950420379639,
+      "learning_rate": 0.0003,
+      "loss": 3.0959878540039063,
+      "step": 2850
+    },
+    {
+      "epoch": 0.1847545631191667,
+      "grad_norm": 0.7572550177574158,
+      "learning_rate": 0.0003,
+      "loss": 3.0922555541992187,
+      "step": 2900
+    },
+    {
+      "epoch": 0.1879399866212213,
+      "grad_norm": 0.7447418570518494,
+      "learning_rate": 0.0003,
+      "loss": 3.092269287109375,
+      "step": 2950
+    },
+    {
+      "epoch": 0.19112541012327589,
+      "grad_norm": 0.7423698306083679,
+      "learning_rate": 0.0003,
+      "loss": 3.0631478881835936,
+      "step": 3000
+    },
+    {
+      "epoch": 0.19112541012327589,
+      "eval_loss": 1.5422803163528442,
+      "eval_runtime": 23.4449,
+      "eval_samples_per_second": 430.925,
+      "eval_steps_per_second": 13.478,
+      "step": 3000
+    },
+    {
+      "epoch": 0.1943108336253305,
+      "grad_norm": 0.7148990035057068,
+      "learning_rate": 0.0003,
+      "loss": 3.0585479736328125,
+      "step": 3050
+    },
+    {
+      "epoch": 0.19749625712738508,
+      "grad_norm": 0.7321158051490784,
+      "learning_rate": 0.0003,
+      "loss": 3.066180419921875,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2006816806294397,
+      "grad_norm": 0.7416279911994934,
+      "learning_rate": 0.0003,
+      "loss": 3.0487716674804686,
+      "step": 3150
+    },
+    {
+      "epoch": 0.20386710413149428,
+      "grad_norm": 0.7152788043022156,
+      "learning_rate": 0.0003,
+      "loss": 3.06279296875,
+      "step": 3200
+    },
+    {
+      "epoch": 0.2070525276335489,
+      "grad_norm": 0.7455261945724487,
+      "learning_rate": 0.0003,
+      "loss": 3.0631439208984377,
+      "step": 3250
+    },
+    {
+      "epoch": 0.21023795113560348,
+      "grad_norm": 0.7299383878707886,
+      "learning_rate": 0.0003,
+      "loss": 3.04324951171875,
+      "step": 3300
+    },
+    {
+      "epoch": 0.21342337463765806,
+      "grad_norm": 0.73154217004776,
+      "learning_rate": 0.0003,
+      "loss": 3.0362890625,
+      "step": 3350
+    },
+    {
+      "epoch": 0.21660879813971268,
+      "grad_norm": 0.7346115708351135,
+      "learning_rate": 0.0003,
+      "loss": 3.024760437011719,
+      "step": 3400
+    },
+    {
+      "epoch": 0.21979422164176726,
+      "grad_norm": 0.7139434218406677,
+      "learning_rate": 0.0003,
+      "loss": 3.0128494262695313,
+      "step": 3450
+    },
+    {
+      "epoch": 0.22297964514382188,
+      "grad_norm": 0.7423503398895264,
+      "learning_rate": 0.0003,
+      "loss": 3.0362188720703127,
+      "step": 3500
+    },
+    {
+      "epoch": 0.22297964514382188,
+      "eval_loss": 1.510373830795288,
+      "eval_runtime": 23.4012,
+      "eval_samples_per_second": 431.73,
+      "eval_steps_per_second": 13.504,
+      "step": 3500
+    },
+    {
+      "epoch": 0.22616506864587646,
+      "grad_norm": 0.7142049074172974,
+      "learning_rate": 0.0003,
+      "loss": 3.013752136230469,
+      "step": 3550
+    },
+    {
+      "epoch": 0.22935049214793107,
+      "grad_norm": 0.7133361101150513,
+      "learning_rate": 0.0003,
+      "loss": 3.0155905151367186,
+      "step": 3600
+    },
+    {
+      "epoch": 0.23253591564998566,
+      "grad_norm": 0.7242637872695923,
+      "learning_rate": 0.0003,
+      "loss": 3.0050390625,
+      "step": 3650
+    },
+    {
+      "epoch": 0.23572133915204027,
+      "grad_norm": 0.7068458199501038,
+      "learning_rate": 0.0003,
+      "loss": 2.997817077636719,
+      "step": 3700
+    },
+    {
+      "epoch": 0.23890676265409486,
+      "grad_norm": 0.7191183567047119,
+      "learning_rate": 0.0003,
+      "loss": 3.0074972534179687,
+      "step": 3750
+    },
+    {
+      "epoch": 0.24209218615614947,
+      "grad_norm": 0.7113999128341675,
+      "learning_rate": 0.0003,
+      "loss": 2.9924588012695312,
+      "step": 3800
+    },
+    {
+      "epoch": 0.24527760965820405,
+      "grad_norm": 0.704169750213623,
+      "learning_rate": 0.0003,
+      "loss": 2.9923251342773436,
+      "step": 3850
+    },
+    {
+      "epoch": 0.24846303316025867,
+      "grad_norm": 0.689486026763916,
+      "learning_rate": 0.0003,
+      "loss": 2.962857971191406,
+      "step": 3900
+    },
+    {
+      "epoch": 0.2516484566623133,
+      "grad_norm": 0.7115288376808167,
+      "learning_rate": 0.0003,
+      "loss": 2.9723941040039064,
+      "step": 3950
+    },
+    {
+      "epoch": 0.25483388016436787,
+      "grad_norm": 0.6829734444618225,
+      "learning_rate": 0.0003,
+      "loss": 2.9560333251953126,
+      "step": 4000
+    },
+    {
+      "epoch": 0.25483388016436787,
+      "eval_loss": 1.482374668121338,
+      "eval_runtime": 22.2808,
+      "eval_samples_per_second": 453.439,
+      "eval_steps_per_second": 14.183,
+      "step": 4000
+    },
+    {
+      "epoch": 0.25801930366642245,
+      "grad_norm": 0.6945831179618835,
+      "learning_rate": 0.0003,
+      "loss": 2.9535784912109375,
+      "step": 4050
+    },
+    {
+      "epoch": 0.26120472716847704,
+      "grad_norm": 0.690827488899231,
+      "learning_rate": 0.0003,
+      "loss": 2.9315560913085936,
+      "step": 4100
+    },
+    {
+      "epoch": 0.2643901506705316,
+      "grad_norm": 0.7251810431480408,
+      "learning_rate": 0.0003,
+      "loss": 2.9491262817382813,
+      "step": 4150
+    },
+    {
+      "epoch": 0.26757557417258626,
+      "grad_norm": 0.6903426647186279,
+      "learning_rate": 0.0003,
+      "loss": 2.9596847534179687,
+      "step": 4200
+    },
+    {
+      "epoch": 0.27076099767464085,
+      "grad_norm": 0.7182983756065369,
+      "learning_rate": 0.0003,
+      "loss": 2.9524960327148437,
+      "step": 4250
+    },
+    {
+      "epoch": 0.27394642117669543,
+      "grad_norm": 0.6987735629081726,
+      "learning_rate": 0.0003,
+      "loss": 2.91990234375,
+      "step": 4300
+    },
+    {
+      "epoch": 0.27713184467875,
+      "grad_norm": 0.7041905522346497,
+      "learning_rate": 0.0003,
+      "loss": 2.9213037109375,
+      "step": 4350
+    },
+    {
+      "epoch": 0.28031726818080466,
+      "grad_norm": 0.7060105800628662,
+      "learning_rate": 0.0003,
+      "loss": 2.949046936035156,
+      "step": 4400
+    },
+    {
+      "epoch": 0.28350269168285924,
+      "grad_norm": 0.6817911863327026,
+      "learning_rate": 0.0003,
+      "loss": 2.9220065307617187,
+      "step": 4450
+    },
+    {
+      "epoch": 0.2866881151849138,
+      "grad_norm": 0.6754902601242065,
+      "learning_rate": 0.0003,
+      "loss": 2.9083685302734374,
+      "step": 4500
+    },
+    {
+      "epoch": 0.2866881151849138,
+      "eval_loss": 1.4599963426589966,
+      "eval_runtime": 23.4291,
+      "eval_samples_per_second": 431.216,
+      "eval_steps_per_second": 13.488,
+      "step": 4500
+    },
+    {
+      "epoch": 0.2898735386869684,
+      "grad_norm": 0.6705558896064758,
+      "learning_rate": 0.0003,
+      "loss": 2.9190487670898437,
+      "step": 4550
+    },
+    {
+      "epoch": 0.29305896218902305,
+      "grad_norm": 0.6795514822006226,
+      "learning_rate": 0.0003,
+      "loss": 2.9107174682617187,
+      "step": 4600
+    },
+    {
+      "epoch": 0.29624438569107764,
+      "grad_norm": 0.6799351572990417,
+      "learning_rate": 0.0003,
+      "loss": 2.9119940185546875,
+      "step": 4650
+    },
+    {
+      "epoch": 0.2994298091931322,
+      "grad_norm": 0.6832090616226196,
+      "learning_rate": 0.0003,
+      "loss": 2.895616455078125,
+      "step": 4700
+    },
+    {
+      "epoch": 0.3026152326951868,
+      "grad_norm": 0.6919124722480774,
+      "learning_rate": 0.0003,
+      "loss": 2.8978277587890626,
+      "step": 4750
+    },
+    {
+      "epoch": 0.30580065619724145,
+      "grad_norm": 0.6905309557914734,
+      "learning_rate": 0.0003,
+      "loss": 2.903988952636719,
+      "step": 4800
+    },
+    {
+      "epoch": 0.30898607969929603,
+      "grad_norm": 0.6617740392684937,
+      "learning_rate": 0.0003,
+      "loss": 2.892375183105469,
+      "step": 4850
+    },
+    {
+      "epoch": 0.3121715032013506,
+      "grad_norm": 0.6902974843978882,
+      "learning_rate": 0.0003,
+      "loss": 2.8707626342773436,
+      "step": 4900
+    },
+    {
+      "epoch": 0.3153569267034052,
+      "grad_norm": 0.6690743565559387,
+      "learning_rate": 0.0003,
+      "loss": 2.8908343505859375,
+      "step": 4950
+    },
+    {
+      "epoch": 0.3185423502054598,
+      "grad_norm": 0.6480445861816406,
+      "learning_rate": 0.0003,
+      "loss": 2.8723358154296874,
+      "step": 5000
+    },
+    {
+      "epoch": 0.3185423502054598,
+      "eval_loss": 1.4423511028289795,
+      "eval_runtime": 22.1542,
+      "eval_samples_per_second": 456.03,
+      "eval_steps_per_second": 14.264,
+      "step": 5000
+    },
+    {
+      "epoch": 0.32172777370751443,
+      "grad_norm": 0.6830846071243286,
+      "learning_rate": 0.0003,
+      "loss": 2.8886477661132814,
+      "step": 5050
+    },
+    {
+      "epoch": 0.324913197209569,
+      "grad_norm": 0.6849225759506226,
+      "learning_rate": 0.0003,
+      "loss": 2.87338134765625,
+      "step": 5100
+    },
+    {
+      "epoch": 0.3280986207116236,
+      "grad_norm": 0.6755518317222595,
+      "learning_rate": 0.0003,
+      "loss": 2.877676696777344,
+      "step": 5150
+    },
+    {
+      "epoch": 0.3312840442136782,
+      "grad_norm": 0.6617788076400757,
+      "learning_rate": 0.0003,
+      "loss": 2.8757110595703126,
+      "step": 5200
+    },
+    {
+      "epoch": 0.3344694677157328,
+      "grad_norm": 0.7036768794059753,
+      "learning_rate": 0.0003,
+      "loss": 2.8573074340820312,
+      "step": 5250
+    },
+    {
+      "epoch": 0.3376548912177874,
+      "grad_norm": 0.6579094529151917,
+      "learning_rate": 0.0003,
+      "loss": 2.8720626831054688,
+      "step": 5300
+    },
+    {
+      "epoch": 0.340840314719842,
+      "grad_norm": 0.6679812073707581,
+      "learning_rate": 0.0003,
+      "loss": 2.851099853515625,
+      "step": 5350
+    },
+    {
+      "epoch": 0.3440257382218966,
+      "grad_norm": 0.6677426695823669,
+      "learning_rate": 0.0003,
+      "loss": 2.8569485473632814,
+      "step": 5400
+    },
+    {
+      "epoch": 0.3472111617239512,
+      "grad_norm": 0.6569070219993591,
+      "learning_rate": 0.0003,
+      "loss": 2.8666485595703124,
+      "step": 5450
+    },
+    {
+      "epoch": 0.3503965852260058,
+      "grad_norm": 0.6604616045951843,
+      "learning_rate": 0.0003,
+      "loss": 2.851506652832031,
+      "step": 5500
+    },
+    {
+      "epoch": 0.3503965852260058,
+      "eval_loss": 1.425577998161316,
+      "eval_runtime": 23.4594,
+      "eval_samples_per_second": 430.659,
+      "eval_steps_per_second": 13.47,
+      "step": 5500
+    },
+    {
+      "epoch": 0.3535820087280604,
+      "grad_norm": 0.681241512298584,
+      "learning_rate": 0.0003,
+      "loss": 2.851204528808594,
+      "step": 5550
+    },
+    {
+      "epoch": 0.356767432230115,
+      "grad_norm": 0.6741161942481995,
+      "learning_rate": 0.0003,
+      "loss": 2.8326242065429685,
+      "step": 5600
+    },
+    {
+      "epoch": 0.3599528557321696,
+      "grad_norm": 0.6524420380592346,
+      "learning_rate": 0.0003,
+      "loss": 2.83824951171875,
+      "step": 5650
+    },
+    {
+      "epoch": 0.3631382792342242,
+      "grad_norm": 0.689453125,
+      "learning_rate": 0.0003,
+      "loss": 2.835894775390625,
+      "step": 5700
+    },
+    {
+      "epoch": 0.3663237027362788,
+      "grad_norm": 0.6474650502204895,
+      "learning_rate": 0.0003,
+      "loss": 2.8316229248046874,
+      "step": 5750
+    },
+    {
+      "epoch": 0.3695091262383334,
+      "grad_norm": 0.6406029462814331,
+      "learning_rate": 0.0003,
+      "loss": 2.830424499511719,
+      "step": 5800
+    },
+    {
+      "epoch": 0.37269454974038796,
+      "grad_norm": 0.6752877235412598,
+      "learning_rate": 0.0003,
+      "loss": 2.8113082885742187,
+      "step": 5850
+    },
+    {
+      "epoch": 0.3758799732424426,
+      "grad_norm": 0.6765608191490173,
+      "learning_rate": 0.0003,
+      "loss": 2.8041595458984374,
+      "step": 5900
+    },
+    {
+      "epoch": 0.3790653967444972,
+      "grad_norm": 0.6739156246185303,
+      "learning_rate": 0.0003,
+      "loss": 2.8140625,
+      "step": 5950
+    },
+    {
+      "epoch": 0.38225082024655177,
+      "grad_norm": 0.6608895659446716,
+      "learning_rate": 0.0003,
+      "loss": 2.81939453125,
+      "step": 6000
+    },
+    {
+      "epoch": 0.38225082024655177,
+      "eval_loss": 1.412307858467102,
+      "eval_runtime": 21.5332,
+      "eval_samples_per_second": 469.181,
+      "eval_steps_per_second": 14.675,
+      "step": 6000
+    },
+    {
+      "epoch": 0.38543624374860636,
+      "grad_norm": 0.6547248363494873,
+      "learning_rate": 0.0003,
+      "loss": 2.806842041015625,
+      "step": 6050
+    },
+    {
+      "epoch": 0.388621667250661,
+      "grad_norm": 0.6861343383789062,
+      "learning_rate": 0.0003,
+      "loss": 2.7924771118164062,
+      "step": 6100
+    },
+    {
+      "epoch": 0.3918070907527156,
+      "grad_norm": 0.6477441191673279,
+      "learning_rate": 0.0003,
+      "loss": 2.803730773925781,
+      "step": 6150
+    },
+    {
+      "epoch": 0.39499251425477017,
+      "grad_norm": 0.6472424864768982,
+      "learning_rate": 0.0003,
+      "loss": 2.797716064453125,
+      "step": 6200
+    },
+    {
+      "epoch": 0.39817793775682475,
+      "grad_norm": 0.6462525129318237,
+      "learning_rate": 0.0003,
+      "loss": 2.8159149169921873,
+      "step": 6250
+    },
+    {
+      "epoch": 0.4013633612588794,
+      "grad_norm": 0.6314276456832886,
+      "learning_rate": 0.0003,
+      "loss": 2.8154486083984374,
+      "step": 6300
+    },
+    {
+      "epoch": 0.404548784760934,
+      "grad_norm": 0.6458173990249634,
+      "learning_rate": 0.0003,
+      "loss": 2.7983084106445313,
+      "step": 6350
+    },
+    {
+      "epoch": 0.40773420826298856,
+      "grad_norm": 0.638751208782196,
+      "learning_rate": 0.0003,
+      "loss": 2.7921502685546873,
+      "step": 6400
+    },
+    {
+      "epoch": 0.41091963176504315,
+      "grad_norm": 0.6702314615249634,
+      "learning_rate": 0.0003,
+      "loss": 2.7908941650390626,
+      "step": 6450
+    },
+    {
+      "epoch": 0.4141050552670978,
+      "grad_norm": 0.642062246799469,
+      "learning_rate": 0.0003,
+      "loss": 2.7946160888671874,
+      "step": 6500
+    },
+    {
+      "epoch": 0.4141050552670978,
+      "eval_loss": 1.3995262384414673,
+      "eval_runtime": 23.7379,
+      "eval_samples_per_second": 425.606,
+      "eval_steps_per_second": 13.312,
+      "step": 6500
+    },
+    {
+      "epoch": 0.4172904787691524,
+      "grad_norm": 0.6468133926391602,
+      "learning_rate": 0.0003,
+      "loss": 2.7932785034179686,
+      "step": 6550
+    },
+    {
+      "epoch": 0.42047590227120696,
+      "grad_norm": 0.6400574445724487,
+      "learning_rate": 0.0003,
+      "loss": 2.797884521484375,
+      "step": 6600
+    },
+    {
+      "epoch": 0.42366132577326154,
+      "grad_norm": 0.6529418230056763,
+      "learning_rate": 0.0003,
+      "loss": 2.787747802734375,
+      "step": 6650
+    },
+    {
+      "epoch": 0.42684674927531613,
+      "grad_norm": 0.6661109924316406,
+      "learning_rate": 0.0003,
+      "loss": 2.7549853515625,
+      "step": 6700
+    },
+    {
+      "epoch": 0.43003217277737077,
+      "grad_norm": 0.630394458770752,
+      "learning_rate": 0.0003,
+      "loss": 2.7755743408203126,
+      "step": 6750
+    },
+    {
+      "epoch": 0.43321759627942535,
+      "grad_norm": 0.6382555961608887,
+      "learning_rate": 0.0003,
+      "loss": 2.7796176147460936,
+      "step": 6800
+    },
+    {
+      "epoch": 0.43640301978147994,
+      "grad_norm": 0.6238535642623901,
+      "learning_rate": 0.0003,
+      "loss": 2.7695758056640627,
+      "step": 6850
+    },
+    {
+      "epoch": 0.4395884432835345,
+      "grad_norm": 0.6449209451675415,
+      "learning_rate": 0.0003,
+      "loss": 2.779901428222656,
+      "step": 6900
+    },
+    {
+      "epoch": 0.44277386678558917,
+      "grad_norm": 0.6255211234092712,
+      "learning_rate": 0.0003,
+      "loss": 2.777834167480469,
+      "step": 6950
+    },
+    {
+      "epoch": 0.44595929028764375,
+      "grad_norm": 0.6496977210044861,
+      "learning_rate": 0.0003,
+      "loss": 2.77800048828125,
+      "step": 7000
+    },
+    {
+      "epoch": 0.44595929028764375,
+      "eval_loss": 1.3886700868606567,
+      "eval_runtime": 22.498,
+      "eval_samples_per_second": 449.063,
+      "eval_steps_per_second": 14.046,
+      "step": 7000
+    },
+    {
+      "epoch": 0.44914471378969834,
+      "grad_norm": 0.639970600605011,
+      "learning_rate": 0.0003,
+      "loss": 2.7682064819335936,
+      "step": 7050
+    },
+    {
+      "epoch": 0.4523301372917529,
+      "grad_norm": 0.6330003142356873,
+      "learning_rate": 0.0003,
+      "loss": 2.761827392578125,
+      "step": 7100
+    },
+    {
+      "epoch": 0.45551556079380756,
+      "grad_norm": 0.6850003600120544,
+      "learning_rate": 0.0003,
+      "loss": 2.752735900878906,
+      "step": 7150
+    },
+    {
+      "epoch": 0.45870098429586215,
+      "grad_norm": 0.6478457450866699,
+      "learning_rate": 0.0003,
+      "loss": 2.7455511474609375,
+      "step": 7200
+    },
+    {
+      "epoch": 0.46188640779791673,
+      "grad_norm": 0.6256737112998962,
+      "learning_rate": 0.0003,
+      "loss": 2.747308349609375,
+      "step": 7250
+    },
+    {
+      "epoch": 0.4650718312999713,
+      "grad_norm": 0.6364322304725647,
+      "learning_rate": 0.0003,
+      "loss": 2.768023681640625,
+      "step": 7300
+    },
+    {
+      "epoch": 0.4682572548020259,
+      "grad_norm": 0.628696858882904,
+      "learning_rate": 0.0003,
+      "loss": 2.7501296997070312,
+      "step": 7350
+    },
+    {
+      "epoch": 0.47144267830408054,
+      "grad_norm": 0.6258370280265808,
+      "learning_rate": 0.0003,
+      "loss": 2.751533203125,
+      "step": 7400
+    },
+    {
+      "epoch": 0.4746281018061351,
+      "grad_norm": 0.6456780433654785,
+      "learning_rate": 0.0003,
+      "loss": 2.755550537109375,
+      "step": 7450
+    },
+    {
+      "epoch": 0.4778135253081897,
+      "grad_norm": 0.6306030750274658,
+      "learning_rate": 0.0003,
+      "loss": 2.7546148681640625,
+      "step": 7500
+    },
+    {
+      "epoch": 0.4778135253081897,
+      "eval_loss": 1.3773651123046875,
+      "eval_runtime": 23.0878,
+      "eval_samples_per_second": 437.59,
+      "eval_steps_per_second": 13.687,
+      "step": 7500
+    },
+    {
+      "epoch": 0.4809989488102443,
+      "grad_norm": 0.6364194750785828,
+      "learning_rate": 0.0003,
+      "loss": 2.7301571655273436,
+      "step": 7550
+    },
+    {
+      "epoch": 0.48418437231229894,
+      "grad_norm": 0.640110433101654,
+      "learning_rate": 0.0003,
+      "loss": 2.7487637329101564,
+      "step": 7600
+    },
+    {
+      "epoch": 0.4873697958143535,
+      "grad_norm": 0.6385582685470581,
+      "learning_rate": 0.0003,
+      "loss": 2.752929382324219,
+      "step": 7650
+    },
+    {
+      "epoch": 0.4905552193164081,
+      "grad_norm": 0.6104156374931335,
+      "learning_rate": 0.0003,
+      "loss": 2.755207824707031,
+      "step": 7700
+    },
+    {
+      "epoch": 0.4937406428184627,
+      "grad_norm": 0.6292197108268738,
+      "learning_rate": 0.0003,
+      "loss": 2.7580880737304687,
+      "step": 7750
+    },
+    {
+      "epoch": 0.49692606632051733,
+      "grad_norm": 0.6293047666549683,
+      "learning_rate": 0.0003,
+      "loss": 2.733714904785156,
+      "step": 7800
+    },
+    {
+      "epoch": 0.5001114898225719,
+      "grad_norm": 0.6153090596199036,
+      "learning_rate": 0.0003,
+      "loss": 2.743830261230469,
+      "step": 7850
+    },
+    {
+      "epoch": 0.5032969133246266,
+      "grad_norm": 0.6130385398864746,
+      "learning_rate": 0.0003,
+      "loss": 2.7247064208984373,
+      "step": 7900
+    },
+    {
+      "epoch": 0.5064823368266811,
+      "grad_norm": 0.621226966381073,
+      "learning_rate": 0.0003,
+      "loss": 2.7473419189453123,
+      "step": 7950
+    },
+    {
+      "epoch": 0.5096677603287357,
+      "grad_norm": 0.6221226453781128,
+      "learning_rate": 0.0003,
+      "loss": 2.7302447509765626,
+      "step": 8000
+    },
+    {
+      "epoch": 0.5096677603287357,
+      "eval_loss": 1.3685990571975708,
+      "eval_runtime": 23.4368,
+      "eval_samples_per_second": 431.075,
+      "eval_steps_per_second": 13.483,
+      "step": 8000
+    },
+    {
+      "epoch": 0.5128531838307903,
+      "grad_norm": 0.6214672327041626,
+      "learning_rate": 0.0003,
+      "loss": 2.726623840332031,
+      "step": 8050
+    },
+    {
+      "epoch": 0.5160386073328449,
+      "grad_norm": 0.6271846890449524,
+      "learning_rate": 0.0003,
+      "loss": 2.722579040527344,
+      "step": 8100
+    },
+    {
+      "epoch": 0.5192240308348995,
+      "grad_norm": 0.6163957715034485,
+      "learning_rate": 0.0003,
+      "loss": 2.7232086181640627,
+      "step": 8150
+    },
+    {
+      "epoch": 0.5224094543369541,
+      "grad_norm": 0.6377554535865784,
+      "learning_rate": 0.0003,
+      "loss": 2.731226806640625,
+      "step": 8200
+    },
+    {
+      "epoch": 0.5255948778390087,
+      "grad_norm": 0.6154534816741943,
+      "learning_rate": 0.0003,
+      "loss": 2.7162045288085936,
+      "step": 8250
+    },
+    {
+      "epoch": 0.5287803013410632,
+      "grad_norm": 0.638851523399353,
+      "learning_rate": 0.0003,
+      "loss": 2.7193792724609374,
+      "step": 8300
+    },
+    {
+      "epoch": 0.5319657248431179,
+      "grad_norm": 0.6041390895843506,
+      "learning_rate": 0.0003,
+      "loss": 2.7182553100585936,
+      "step": 8350
+    },
+    {
+      "epoch": 0.5351511483451725,
+      "grad_norm": 0.6189262866973877,
+      "learning_rate": 0.0003,
+      "loss": 2.7203045654296876,
+      "step": 8400
+    },
+    {
+      "epoch": 0.538336571847227,
+      "grad_norm": 0.600339412689209,
+      "learning_rate": 0.0003,
+      "loss": 2.706983947753906,
+      "step": 8450
+    },
+    {
+      "epoch": 0.5415219953492817,
+      "grad_norm": 0.6224169731140137,
+      "learning_rate": 0.0003,
+      "loss": 2.7196090698242186,
+      "step": 8500
+    },
+    {
+      "epoch": 0.5415219953492817,
+      "eval_loss": 1.3598597049713135,
+      "eval_runtime": 21.8633,
+      "eval_samples_per_second": 462.099,
+      "eval_steps_per_second": 14.453,
+      "step": 8500
+    },
+    {
+      "epoch": 0.5447074188513363,
+      "grad_norm": 0.6039189100265503,
+      "learning_rate": 0.0003,
+      "loss": 2.707725830078125,
+      "step": 8550
+    },
+    {
+      "epoch": 0.5478928423533909,
+      "grad_norm": 0.614460289478302,
+      "learning_rate": 0.0003,
+      "loss": 2.705989990234375,
+      "step": 8600
+    },
+    {
+      "epoch": 0.5510782658554455,
+      "grad_norm": 0.6237956881523132,
+      "learning_rate": 0.0003,
+      "loss": 2.724997253417969,
+      "step": 8650
+    },
+    {
+      "epoch": 0.5542636893575,
+      "grad_norm": 0.6046378016471863,
+      "learning_rate": 0.0003,
+      "loss": 2.7018728637695313,
+      "step": 8700
+    },
+    {
+      "epoch": 0.5574491128595547,
+      "grad_norm": 0.621566116809845,
+      "learning_rate": 0.0003,
+      "loss": 2.693578796386719,
+      "step": 8750
+    },
+    {
+      "epoch": 0.5606345363616093,
+      "grad_norm": 0.611810028553009,
+      "learning_rate": 0.0003,
+      "loss": 2.7023934936523437,
+      "step": 8800
+    },
+    {
+      "epoch": 0.5638199598636638,
+      "grad_norm": 0.6239098906517029,
+      "learning_rate": 0.0003,
+      "loss": 2.705491638183594,
+      "step": 8850
+    },
+    {
+      "epoch": 0.5670053833657185,
+      "grad_norm": 0.6088017225265503,
+      "learning_rate": 0.0003,
+      "loss": 2.6936785888671877,
+      "step": 8900
+    },
+    {
+      "epoch": 0.5701908068677731,
+      "grad_norm": 0.610343337059021,
+      "learning_rate": 0.0003,
+      "loss": 2.7051541137695314,
+      "step": 8950
+    },
+    {
+      "epoch": 0.5733762303698277,
+      "grad_norm": 0.5963529348373413,
+      "learning_rate": 0.0003,
+      "loss": 2.713864440917969,
+      "step": 9000
+    },
+    {
+      "epoch": 0.5733762303698277,
+      "eval_loss": 1.3529787063598633,
+      "eval_runtime": 23.4537,
+      "eval_samples_per_second": 430.764,
+      "eval_steps_per_second": 13.473,
+      "step": 9000
+    },
+    {
+      "epoch": 0.5765616538718823,
+      "grad_norm": 0.6124906539916992,
+      "learning_rate": 0.0003,
+      "loss": 2.690745544433594,
+      "step": 9050
+    },
+    {
+      "epoch": 0.5797470773739368,
+      "grad_norm": 0.6216937899589539,
+      "learning_rate": 0.0003,
+      "loss": 2.699717712402344,
+      "step": 9100
+    },
+    {
+      "epoch": 0.5829325008759915,
+      "grad_norm": 0.5955805778503418,
+      "learning_rate": 0.0003,
+      "loss": 2.6945562744140625,
+      "step": 9150
+    },
+    {
+      "epoch": 0.5861179243780461,
+      "grad_norm": 0.6079633831977844,
+      "learning_rate": 0.0003,
+      "loss": 2.6944207763671875,
+      "step": 9200
+    },
+    {
+      "epoch": 0.5893033478801006,
+      "grad_norm": 0.5997431874275208,
+      "learning_rate": 0.0003,
+      "loss": 2.682657470703125,
+      "step": 9250
+    },
+    {
+      "epoch": 0.5924887713821553,
+      "grad_norm": 0.5978936553001404,
+      "learning_rate": 0.0003,
+      "loss": 2.6979690551757813,
+      "step": 9300
+    },
+    {
+      "epoch": 0.5956741948842098,
+      "grad_norm": 0.6001494526863098,
+      "learning_rate": 0.0003,
+      "loss": 2.678921813964844,
+      "step": 9350
+    },
+    {
+      "epoch": 0.5988596183862644,
+      "grad_norm": 0.6094045639038086,
+      "learning_rate": 0.0003,
+      "loss": 2.6512747192382813,
+      "step": 9400
+    },
+    {
+      "epoch": 0.6020450418883191,
+      "grad_norm": 0.6045228838920593,
+      "learning_rate": 0.0003,
+      "loss": 2.691983642578125,
+      "step": 9450
+    },
+    {
+      "epoch": 0.6052304653903736,
+      "grad_norm": 0.6013676524162292,
+      "learning_rate": 0.0003,
+      "loss": 2.683333435058594,
+      "step": 9500
+    },
+    {
+      "epoch": 0.6052304653903736,
+      "eval_loss": 1.344848394393921,
+      "eval_runtime": 21.6697,
+      "eval_samples_per_second": 466.227,
+      "eval_steps_per_second": 14.583,
+      "step": 9500
+    },
+    {
+      "epoch": 0.6084158888924283,
+      "grad_norm": 0.6003855466842651,
+      "learning_rate": 0.0003,
+      "loss": 2.676940612792969,
+      "step": 9550
+    },
+    {
+      "epoch": 0.6116013123944829,
+      "grad_norm": 0.6023901104927063,
+      "learning_rate": 0.0003,
+      "loss": 2.68151123046875,
+      "step": 9600
+    },
+    {
+      "epoch": 0.6147867358965374,
+      "grad_norm": 0.5977089405059814,
+      "learning_rate": 0.0003,
+      "loss": 2.6782522583007813,
+      "step": 9650
+    },
+    {
+      "epoch": 0.6179721593985921,
+      "grad_norm": 0.6092996597290039,
+      "learning_rate": 0.0003,
+      "loss": 2.6822268676757814,
+      "step": 9700
+    },
+    {
+      "epoch": 0.6211575829006466,
+      "grad_norm": 0.6068724989891052,
+      "learning_rate": 0.0003,
+      "loss": 2.6791806030273437,
+      "step": 9750
+    },
+    {
+      "epoch": 0.6243430064027012,
+      "grad_norm": 0.6010355949401855,
+      "learning_rate": 0.0003,
+      "loss": 2.674016418457031,
+      "step": 9800
+    },
+    {
+      "epoch": 0.6275284299047559,
+      "grad_norm": 0.597061276435852,
+      "learning_rate": 0.0003,
+      "loss": 2.6548828125,
+      "step": 9850
+    },
+    {
+      "epoch": 0.6307138534068104,
+      "grad_norm": 0.6113420724868774,
+      "learning_rate": 0.0003,
+      "loss": 2.6783132934570313,
+      "step": 9900
+    },
+    {
+      "epoch": 0.633899276908865,
+      "grad_norm": 0.5893495082855225,
+      "learning_rate": 0.0003,
+      "loss": 2.669444580078125,
+      "step": 9950
+    },
+    {
+      "epoch": 0.6370847004109196,
+      "grad_norm": 0.6023650169372559,
+      "learning_rate": 0.0003,
+      "loss": 2.6637033081054686,
+      "step": 10000
+    },
+    {
+      "epoch": 0.6370847004109196,
+      "eval_loss": 1.3375693559646606,
+      "eval_runtime": 23.3102,
+      "eval_samples_per_second": 433.415,
+      "eval_steps_per_second": 13.556,
+      "step": 10000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.01270851584e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

out/gelu_run/checkpoint-10000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fa1a0b0e5363cc36836421f3897c70be29fbcdf36b446a98ed1a52d6ba697a2
+size 4856

out/gelu_run/config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "glu_activation": "gelu",
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "tiny_llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

out/gelu_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db6e127711dc7a082314a726aa5990adf7c2d5d2be930edd888d5efe90fa2260
+size 208226064

out/gelu_run/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

out/gelu_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fa1a0b0e5363cc36836421f3897c70be29fbcdf36b446a98ed1a52d6ba697a2
+size 4856

out/gelu_run/training_log.jsonl

@@ -215,3 +215,8 @@
 {"step": 9800, "epoch": 0.6243430064027012, "timestamp": 1785791003.7711053, "loss": 2.674016418457031, "grad_norm": 0.6010355949401855, "learning_rate": 0.0003, "train/total_time_seconds": 2743.781830396503, "train/time_per_step_avg": 0.2649773460626602, "train/epoch_time_elapsed": 3673.754510141909, "train/estimated_remaining_minutes": 0.9332591259852052}
 {"step": 9850, "epoch": 0.6275284299047559, "timestamp": 1785791020.7603729, "loss": 2.6548828125, "grad_norm": 0.597061276435852, "learning_rate": 0.0003, "train/total_time_seconds": 2758.3340581022203, "train/time_per_step_avg": 0.28878977343440054, "train/epoch_time_elapsed": 3690.74377720803, "train/estimated_remaining_minutes": 0.7000847863203604}
 {"step": 9900, "epoch": 0.6307138534068104, "timestamp": 1785791037.5630002, "loss": 2.6783132934570313, "grad_norm": 0.6113420724868774, "learning_rate": 0.0003, "train/total_time_seconds": 2772.637024130672, "train/time_per_step_avg": 0.28855193734169005, "train/epoch_time_elapsed": 3707.5464034341276, "train/estimated_remaining_minutes": 0.46677390978630845}
+{"step": 9950, "epoch": 0.633899276908865, "timestamp": 1785791053.6952314, "loss": 2.669444580078125, "grad_norm": 0.5893495082855225, "learning_rate": 0.0003, "train/total_time_seconds": 2786.329146616161, "train/time_per_step_avg": 0.27995088513940575, "train/epoch_time_elapsed": 3723.678635638207, "train/estimated_remaining_minutes": 0.23336090005160476}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785791069.38644, "loss": 2.6637033081054686, "grad_norm": 0.6023650169372559, "learning_rate": 0.0003, "train/total_time_seconds": 2799.2567448094487, "train/time_per_step_avg": 0.26619720678776504, "train/epoch_time_elapsed": 3739.3698446899652, "train/estimated_remaining_minutes": 0.0}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785791092.6981473, "eval_loss": 1.3375693559646606, "eval_runtime": 23.3102, "eval_samples_per_second": 433.415, "eval_steps_per_second": 13.556, "train/total_time_seconds": 2799.2567448094487, "train/time_per_step_avg": 0.26619720678776504, "train/epoch_time_elapsed": 3762.681552838534, "train/estimated_remaining_minutes": 0.0}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785791093.4411416, "train_runtime": 3763.4284, "train_samples_per_second": 170.058, "train_steps_per_second": 2.657, "total_flos": 7.01270851584e+16, "train_loss": 3.200245819091797, "train/total_time_seconds": 2799.2567448094487, "train/time_per_step_avg": 0.26619720678776504, "train/epoch_time_elapsed": 3763.4245455749333, "train/estimated_remaining_minutes": 0.0}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785791121.4517596, "eval_loss": 1.3375693559646606, "eval_runtime": 28.0075, "eval_samples_per_second": 360.725, "eval_steps_per_second": 11.283, "train/total_time_seconds": 2799.2567448094487, "train/time_per_step_avg": 0.26619720678776504, "train/epoch_time_elapsed": 3791.4351624287665, "train/estimated_remaining_minutes": 0.0}

wandb/debug-internal.log

@@ -977,3 +977,15 @@
 {"time":"2026-08-03T21:03:37.904023512Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-03T21:03:52.618668283Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":437,"history_lines":1,"events_offset":994,"events_lines":2,"console_offset":600,"console_lines":1}
 {"time":"2026-08-03T21:03:52.863015286Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:07.617848127Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":438,"history_lines":1,"events_offset":996,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:04:07.873552069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:22.618432846Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":439,"history_lines":1,"events_offset":998,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:04:22.845182521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:37.61809596Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":440,"history_lines":1,"events_offset":1000,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:04:37.932705653Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:52.617961886Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1002,"events_lines":2,"console_offset":611,"console_lines":12}
+{"time":"2026-08-03T21:04:52.848518768Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:05:07.618347372Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":441,"history_lines":2,"events_offset":1004,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:05:07.911226489Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:05:22.624347304Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":443,"history_lines":1,"events_offset":1006,"events_lines":2,"console_offset":611,"console_lines":1}
+{"time":"2026-08-03T21:05:22.851501949Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

wandb/debug.log

@@ -24,3 +24,6 @@ config: {'_wandb': {}}
 2026-08-03 20:02:10,014 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 16, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'gelu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/gelu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 50, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 500, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-3-gelu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-03 20:02:10,016 INFO    MainThread:2875822 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 52052480 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c10d603910>>
 2026-08-03 20:02:10,016 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 52052480 None
+2026-08-03 21:05:27,837 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 16, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'relu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/relu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 50, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 500, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-3-relu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-03 21:05:27,841 INFO    MainThread:2875822 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 52052480 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c10d603910>>
+2026-08-03 21:05:27,841 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 52052480 None

wandb/run-20260803_185911-6gj59z3n/files/output.log

@@ -598,7 +598,7 @@ Writing model shards: 100%|█████████████████
 Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  4.18it/s]
 /mnt/data/zainulabideen/zain-exp/notebooks/2/Activation/exp.py:416: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at /pytorch/aten/src/ATen/native/ReduceOps.cpp:1831.)
   "std": tensor.std().item(),
- 99%|██████████████████████████████████▋| 9900/10000 [1:01:47<00:33,  2.98it/s]
+100%|██████████████████████████████████| 10000/10000 [1:02:42<00:00,  2.91it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '2.691', 'grad_norm': '0.6125', 'learning_rate': '0.0003', 'epoch': '0.5766', 'train/total_time_seconds': '2535', 'train/time_per_step_avg': '0.2939', 'train/epoch_time_elapsed': '3406', 'train/estimated_remaining_minutes': '4.435', 'train/global/act/norm': '2.298e+05', 'train/global/act/mean': '-1.405', 'train/global/act/std': '3.092', 'train/global/act/max_abs': '61.87', 'train/global/act/frac_near_dtype_limit': '0', 'train/global/act/frac_near_user_limit': '4.052e-08', 'train/global/grad/norm': '0.5897', 'train/global/grad/mean': '4.923e-09', 'train/global/grad/std': '5.779e-05', 'train/global/grad/max_abs': '0.008638', 'train/global/grad/frac_near_dtype_limit': '0', 'train/global/grad/frac_near_user_limit': '0', 'train/global/param/norm': '310.2', 'train/global/param/mean': '0.0001399', 'train/global/param/std': '0.043', 'train/global/param/max_abs': '1.638', 'train/global/param/frac_near_dtype_limit': '0', 'train/global/param/frac_near_user_limit': '0', 'train/layer__model_layers_14/param/norm': '65.16', 'train/layer__model_layers_14/param/mean': '0.0004898', 'train/layer__model_layers_14/param/std': '0.04364', 'train/layer__model_layers_14/param/max_abs': '1.207', 'train/layer__model_layers_14/param/frac_near_dtype_limit': '0', 'train/layer__model_layers_14/param/frac_near_user_limit': '0', 'train/layer_model_layers_0/act/norm': '1.1e+04', 'train/layer_model_layers_0/act/mean': '-0.03693', 'train/layer_model_layers_0/act/std': '0.7439', 'train/layer_model_layers_0/act/max_abs': '15.6', 'train/layer_model_layers_0/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/act/frac_near_user_limit': '0', 'train/layer_model_layers_0/grad/norm': '0.1941', 'train/layer_model_layers_0/grad/mean': '3.247e-08', 'train/layer_model_layers_0/grad/std': '9.191e-05', 'train/layer_model_layers_0/grad/max_abs': '0.005005', 'train/layer_model_layers_0/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/grad/frac_near_user_limit': '0', 'train/layer_model_layers_7/act/norm': '1.316e+04', 'train/layer_model_layers_7/act/mean': '-0.06116', 'train/layer_model_layers_7/act/std': '0.8889', 'train/layer_model_layers_7/act/max_abs': '45.17', 'train/layer_model_layers_7/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/act/frac_near_user_limit': '0', 'train/layer_model_layers_7/grad/norm': '0.1029', 'train/layer_model_layers_7/grad/mean': '3.919e-08', 'train/layer_model_layers_7/grad/std': '4.874e-05', 'train/layer_model_layers_7/grad/max_abs': '0.003815', 'train/layer_model_layers_7/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/grad/frac_near_user_limit': '0', 'train/layer_model_layers_9/act/norm': '1.594e+04', 'train/layer_model_layers_9/act/mean': '-0.04301', 'train/layer_model_layers_9/act/std': '1.079', 'train/layer_model_layers_9/act/max_abs': '43.21', 'train/layer_model_layers_9/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/act/frac_near_user_limit': '0', 'train/layer_model_layers_9/grad/norm': '0.1275', 'train/layer_model_layers_9/grad/mean': '-2.803e-08', 'train/layer_model_layers_9/grad/std': '6.039e-05', 'train/layer_model_layers_9/grad/max_abs': '0.001091', 'train/layer_model_layers_9/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/grad/frac_near_user_limit': '0', 'train/layer_model_layers_8/act/norm': '1.528e+04', 'train/layer_model_layers_8/act/mean': '-0.05503', 'train/layer_model_layers_8/act/std': '1.033', 'train/layer_model_layers_8/act/max_abs': '43.48', 'train/layer_model_layers_8/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/act/frac_near_user_limit': '0', 'train/layer_model_layers_8/grad/norm': '0.1321', 'train/layer_model_layers_8/grad/mean': '-3.506e-08', 'train/layer_model_layers_8/grad/std': '6.254e-05', 'train/layer_model_layers_8/grad/max_abs': '0.001938', 'train/layer_model_layers_8/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/grad/frac_near_user_limit': '0', 'train/layer__model_layers_2/param/norm': '51.66', 'train/layer__model_layers_2/param/mean': '0.000319', 'train/layer__model_layers_2/param/std': '0
 {'loss': '2.7', 'grad_norm': '0.6217', 'learning_rate': '0.0003', 'epoch': '0.5797', 'train/total_time_seconds': '2547', 'train/time_per_step_avg': '0.2719', 'train/epoch_time_elapsed': '3421', 'train/estimated_remaining_minutes': '4.199'}
 {'loss': '2.695', 'grad_norm': '0.5956', 'learning_rate': '0.0003', 'epoch': '0.5829', 'train/total_time_seconds': '2561', 'train/time_per_step_avg': '0.2673', 'train/epoch_time_elapsed': '3437', 'train/estimated_remaining_minutes': '3.966'}
@@ -609,7 +609,7 @@ Writing model shards: 100%|█████████████████
 {'loss': '2.651', 'grad_norm': '0.6094', 'learning_rate': '0.0003', 'epoch': '0.5989', 'train/total_time_seconds': '2631', 'train/time_per_step_avg': '0.2733', 'train/epoch_time_elapsed': '3520', 'train/estimated_remaining_minutes': '2.799'}
 {'loss': '2.692', 'grad_norm': '0.6045', 'learning_rate': '0.0003', 'epoch': '0.602', 'train/total_time_seconds': '2646', 'train/time_per_step_avg': '0.2874', 'train/epoch_time_elapsed': '3536', 'train/estimated_remaining_minutes': '2.566'}
 {'loss': '2.683', 'grad_norm': '0.6014', 'learning_rate': '0.0003', 'epoch': '0.6052', 'train/total_time_seconds': '2660', 'train/time_per_step_avg': '0.287', 'train/epoch_time_elapsed': '3553', 'train/estimated_remaining_minutes': '2.333'}
-                                                                               
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
 {'eval_loss': '1.345', 'eval_runtime': '21.67', 'eval_samples_per_second': '466.2', 'eval_steps_per_second': '14.58', 'epoch': '0.6052', 'train/total_time_seconds': '2660', 'train/time_per_step_avg': '0.287', 'train/epoch_time_elapsed': '3575', 'train/estimated_remaining_minutes': '2.333'}
 {'loss': '2.677', 'grad_norm': '0.6004', 'learning_rate': '0.0003', 'epoch': '0.6084', 'train/total_time_seconds': '2674', 'train/time_per_step_avg': '0.289', 'train/epoch_time_elapsed': '3592', 'train/estimated_remaining_minutes': '2.1', 'train/global/act/norm': '2.327e+05', 'train/global/act/mean': '-1.419', 'train/global/act/std': '3.128', 'train/global/act/max_abs': '69.09', 'train/global/act/frac_near_dtype_limit': '0', 'train/global/act/frac_near_user_limit': '6.383e-08', 'train/global/grad/norm': '0.581', 'train/global/grad/mean': '-1.039e-08', 'train/global/grad/std': '5.694e-05', 'train/global/grad/max_abs': '0.00936', 'train/global/grad/frac_near_dtype_limit': '0', 'train/global/grad/frac_near_user_limit': '0', 'train/global/param/norm': '313', 'train/global/param/mean': '0.0001595', 'train/global/param/std': '0.04338', 'train/global/param/max_abs': '1.647', 'train/global/param/frac_near_dtype_limit': '0', 'train/global/param/frac_near_user_limit': '0', 'train/layer__model_layers_14/param/norm': '65.82', 'train/layer__model_layers_14/param/mean': '0.000486', 'train/layer__model_layers_14/param/std': '0.04408', 'train/layer__model_layers_14/param/max_abs': '1.209', 'train/layer__model_layers_14/param/frac_near_dtype_limit': '0', 'train/layer__model_layers_14/param/frac_near_user_limit': '0', 'train/layer_model_layers_0/act/norm': '1.109e+04', 'train/layer_model_layers_0/act/mean': '-0.03778', 'train/layer_model_layers_0/act/std': '0.7501', 'train/layer_model_layers_0/act/max_abs': '16.01', 'train/layer_model_layers_0/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/act/frac_near_user_limit': '0', 'train/layer_model_layers_0/grad/norm': '0.1893', 'train/layer_model_layers_0/grad/mean': '-2.826e-08', 'train/layer_model_layers_0/grad/std': '8.963e-05', 'train/layer_model_layers_0/grad/max_abs': '0.003632', 'train/layer_model_layers_0/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/grad/frac_near_user_limit': '0', 'train/layer_model_layers_7/act/norm': '1.327e+04', 'train/layer_model_layers_7/act/mean': '-0.05829', 'train/layer_model_layers_7/act/std': '0.897', 'train/layer_model_layers_7/act/max_abs': '49.6', 'train/layer_model_layers_7/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/act/frac_near_user_limit': '0', 'train/layer_model_layers_7/grad/norm': '0.1001', 'train/layer_model_layers_7/grad/mean': '-2.205e-08', 'train/layer_model_layers_7/grad/std': '4.738e-05', 'train/layer_model_layers_7/grad/max_abs': '0.001945', 'train/layer_model_layers_7/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/grad/frac_near_user_limit': '0', 'train/layer_model_layers_9/act/norm': '1.617e+04', 'train/layer_model_layers_9/act/mean': '-0.04216', 'train/layer_model_layers_9/act/std': '1.094', 'train/layer_model_layers_9/act/max_abs': '47.42', 'train/layer_model_layers_9/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/act/frac_near_user_limit': '0', 'train/layer_model_layers_9/grad/norm': '0.1248', 'train/layer_model_layers_9/grad/mean': '2.862e-08', 'train/layer_model_layers_9/grad/std': '5.911e-05', 'train/layer_model_layers_9/grad/max_abs': '0.0009499', 'train/layer_model_layers_9/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/grad/frac_near_user_limit': '0', 'train/layer_model_layers_8/act/norm': '1.547e+04', 'train/layer_model_layers_8/act/mean': '-0.05338', 'train/layer_model_layers_8/act/std': '1.045', 'train/layer_model_layers_8/act/max_abs': '47.89', 'train/layer_model_layers_8/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/act/frac_near_user_limit': '0', 'train/layer_model_layers_8/grad/norm': '0.1289', 'train/layer_model_layers_8/grad/mean': '-3.31e-09', 'train/layer_model_layers_8/grad/std': '6.107e-05', 'train/layer_model_layers_8/grad/max_abs': '0.001251', 'train/layer_model_layers_8/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/grad/frac_near_user_limit': '0', 'train/layer__model_layers_2/param/norm': '52.24', 'train/layer__model_layers_2/param/mean': '0.0003164', 'train/layer__model_layers_2/param/std': '0.0
 {'loss': '2.682', 'grad_norm': '0.6024', 'learning_rate': '0.0003', 'epoch': '0.6116', 'train/total_time_seconds': '2689', 'train/time_per_step_avg': '0.2876', 'train/epoch_time_elapsed': '3609', 'train/estimated_remaining_minutes': '1.867'}
@@ -619,3 +619,55 @@ Writing model shards: 100%|█████████████████
 {'loss': '2.674', 'grad_norm': '0.601', 'learning_rate': '0.0003', 'epoch': '0.6243', 'train/total_time_seconds': '2744', 'train/time_per_step_avg': '0.265', 'train/epoch_time_elapsed': '3674', 'train/estimated_remaining_minutes': '0.9333'}
 {'loss': '2.655', 'grad_norm': '0.5971', 'learning_rate': '0.0003', 'epoch': '0.6275', 'train/total_time_seconds': '2758', 'train/time_per_step_avg': '0.2888', 'train/epoch_time_elapsed': '3691', 'train/estimated_remaining_minutes': '0.7001'}
 {'loss': '2.678', 'grad_norm': '0.6113', 'learning_rate': '0.0003', 'epoch': '0.6307', 'train/total_time_seconds': '2773', 'train/time_per_step_avg': '0.2886', 'train/epoch_time_elapsed': '3708', 'train/estimated_remaining_minutes': '0.4668'}
+{'loss': '2.669', 'grad_norm': '0.5893', 'learning_rate': '0.0003', 'epoch': '0.6339', 'train/total_time_seconds': '2786', 'train/time_per_step_avg': '0.28', 'train/epoch_time_elapsed': '3724', 'train/estimated_remaining_minutes': '0.2334'}
+{'loss': '2.664', 'grad_norm': '0.6024', 'learning_rate': '0.0003', 'epoch': '0.6371', 'train/total_time_seconds': '2799', 'train/time_per_step_avg': '0.2662', 'train/epoch_time_elapsed': '3739', 'train/estimated_remaining_minutes': '0'}
+{'eval_loss': '1.338', 'eval_runtime': '23.31', 'eval_samples_per_second': '433.4', 'eval_steps_per_second': '13.56', 'epoch': '0.6371', 'train/total_time_seconds': '2799', 'train/time_per_step_avg': '0.2662', 'train/epoch_time_elapsed': '3763', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  4.40it/s]
+100%|██████████████████████████████████| 10000/10000 [1:02:43<00:00,  2.66it/s]
+{'train_runtime': '3763', 'train_samples_per_second': '170.1', 'train_steps_per_second': '2.657', 'train_loss': '3.2', 'epoch': '0.6371', 'train/total_time_seconds': '2799', 'train/time_per_step_avg': '0.2662', 'train/epoch_time_elapsed': '3763', 'train/estimated_remaining_minutes': '0'}
+100%|████████████████████████████████████████| 316/316 [00:27<00:00, 11.31it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  3.96it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  2.75it/s]
+Found 8 files to upload
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files  66.6MB · 133MB/s
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files  207MB · 176MB/s
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files  208MB · 124MB/s
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files  208MB · 87.1MB/s ✓
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files  208MB · 60.9MB/s ✓
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files  208MB · 42.7MB/s ✓
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files  208MB · 42.7MB/s ✓
+  Committing  ████████████████████  8 / 8 ✓
+
+============================================================
+>>> Activation: relu | Out: out/relu_run
+============================================================
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.

wandb/run-20260803_185911-6gj59z3n/logs/debug-internal.log

@@ -977,3 +977,15 @@
 {"time":"2026-08-03T21:03:37.904023512Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-03T21:03:52.618668283Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":437,"history_lines":1,"events_offset":994,"events_lines":2,"console_offset":600,"console_lines":1}
 {"time":"2026-08-03T21:03:52.863015286Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:07.617848127Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":438,"history_lines":1,"events_offset":996,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:04:07.873552069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:22.618432846Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":439,"history_lines":1,"events_offset":998,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:04:22.845182521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:37.61809596Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":440,"history_lines":1,"events_offset":1000,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:04:37.932705653Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:04:52.617961886Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":1002,"events_lines":2,"console_offset":611,"console_lines":12}
+{"time":"2026-08-03T21:04:52.848518768Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:05:07.618347372Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":441,"history_lines":2,"events_offset":1004,"events_lines":2,"console_offset":600,"console_lines":1}
+{"time":"2026-08-03T21:05:07.911226489Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T21:05:22.624347304Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":443,"history_lines":1,"events_offset":1006,"events_lines":2,"console_offset":611,"console_lines":1}
+{"time":"2026-08-03T21:05:22.851501949Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

wandb/run-20260803_185911-6gj59z3n/logs/debug.log

@@ -24,3 +24,6 @@ config: {'_wandb': {}}
 2026-08-03 20:02:10,014 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 16, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'gelu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/gelu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 50, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 500, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-3-gelu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-03 20:02:10,016 INFO    MainThread:2875822 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 52052480 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c10d603910>>
 2026-08-03 20:02:10,016 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 52052480 None
+2026-08-03 21:05:27,837 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 16, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'relu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/relu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 50, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 500, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-3-relu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-03 21:05:27,841 INFO    MainThread:2875822 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 52052480 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c10d603910>>
+2026-08-03 21:05:27,841 INFO    MainThread:2875822 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 52052480 None

wandb/run-20260803_185911-6gj59z3n/run-6gj59z3n.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f9d4bacb2dc63a34eddb554da4f16cca386d384a957cc7229127dc84d8ae0fb
-size 11337728
+oid sha256:ec15052245d0cc074ad293855a5e6de8ced85027e34cf1400e72e9c8cbfae372
+size 11501568