Auto upload 2026-08-03T20:22:30.441770

out/gelu_run/README.md

@@ -0,0 +1,97 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+model-index:
+- name: tiny-llama-baseline-gelu
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# tiny-llama-baseline-gelu
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.3872
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 32
+- eval_batch_size: 32
+- seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: constant
+- training_steps: 10000
+
+### Training results
+
+| Training Loss | Epoch  | Step  | Validation Loss |
+|:-------------:|:------:|:-----:|:---------------:|
+| 5.8115        | 0.0159 | 250   | 2.8801          |
+| 4.6262        | 0.0319 | 500   | 2.3344          |
+| 4.1792        | 0.0478 | 750   | 2.0890          |
+| 3.8695        | 0.0637 | 1000  | 1.9427          |
+| 3.7163        | 0.0796 | 1250  | 1.8483          |
+| 3.5153        | 0.0956 | 1500  | 1.7793          |
+| 3.4517        | 0.1115 | 1750  | 1.7311          |
+| 3.3712        | 0.1274 | 2000  | 1.6902          |
+| 3.3071        | 0.1433 | 2250  | 1.6564          |
+| 3.1952        | 0.1593 | 2500  | 1.6281          |
+| 3.1764        | 0.1752 | 2750  | 1.6048          |
+| 3.1841        | 0.1911 | 3000  | 1.5861          |
+| 3.1887        | 0.2071 | 3250  | 1.5709          |
+| 3.1254        | 0.2230 | 3500  | 1.5566          |
+| 3.0764        | 0.2389 | 3750  | 1.5394          |
+| 3.0250        | 0.2548 | 4000  | 1.5282          |
+| 3.0381        | 0.2708 | 4250  | 1.5174          |
+| 2.9927        | 0.2867 | 4500  | 1.5064          |
+| 2.9577        | 0.3026 | 4750  | 1.4978          |
+| 2.9401        | 0.3185 | 5000  | 1.4891          |
+| 2.9238        | 0.3345 | 5250  | 1.4809          |
+| 2.9697        | 0.3504 | 5500  | 1.4734          |
+| 2.9711        | 0.3663 | 5750  | 1.4655          |
+| 2.9360        | 0.3823 | 6000  | 1.4597          |
+| 2.9155        | 0.3982 | 6250  | 1.4540          |
+| 2.8983        | 0.4141 | 6500  | 1.4485          |
+| 2.8937        | 0.4300 | 6750  | 1.4428          |
+| 2.8779        | 0.4460 | 7000  | 1.4374          |
+| 2.8592        | 0.4619 | 7250  | 1.4312          |
+| 2.8556        | 0.4778 | 7500  | 1.4266          |
+| 2.8630        | 0.4937 | 7750  | 1.4223          |
+| 2.7939        | 0.5097 | 8000  | 1.4177          |
+| 2.8342        | 0.5256 | 8250  | 1.4145          |
+| 2.8549        | 0.5415 | 8500  | 1.4098          |
+| 2.8061        | 0.5574 | 8750  | 1.4052          |
+| 2.8270        | 0.5734 | 9000  | 1.4023          |
+| 2.7998        | 0.5893 | 9250  | 1.3985          |
+| 2.7635        | 0.6052 | 9500  | 1.3957          |
+| 2.7969        | 0.6212 | 9750  | 1.3913          |
+| 2.7639        | 0.6371 | 10000 | 1.3872          |
+
+
+### Framework versions
+
+- Transformers 5.15.0.dev0
+- Pytorch 2.6.0+cu124
+- Datasets 5.0.1
+- Tokenizers 0.22.2

out/gelu_run/chat_template.jinja

@@ -0,0 +1,15 @@
+{% for message in messages %}
+{% if message['role'] == 'user' %}
+{{ '<|user|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'system' %}
+{{ '<|system|>
+' + message['content'] + eos_token }}
+{% elif message['role'] == 'assistant' %}
+{{ '<|assistant|>
+'  + message['content'] + eos_token }}
+{% endif %}
+{% if loop.last and add_generation_prompt %}
+{{ '<|assistant|>' }}
+{% endif %}
+{% endfor %}

out/gelu_run/config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "glu_activation": "gelu",
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "tiny_llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

out/gelu_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90e41c500d3a235dd9c9ae4cc37056b507fe2b61ad899b8888c35f01118d0f9d
+size 136882104

out/gelu_run/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

out/gelu_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebdf9f4840c564ec5ae3156213ea83961eee68d24767cb7240b2a86030badf82
+size 4856

out/gelu_run/training_log.jsonl

@@ -1039,3 +1039,4 @@
 {"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785788471.3660119, "loss": 2.7639394760131837, "grad_norm": 0.7221068739891052, "learning_rate": 0.0003, "train/total_time_seconds": 1861.7832489609718, "train/time_per_step_avg": 0.1672628151997924, "train/epoch_time_elapsed": 3059.8260883316398, "train/estimated_remaining_minutes": 0.0}
 {"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785788489.710551, "eval_loss": 1.3872374296188354, "eval_runtime": 18.3257, "eval_samples_per_second": 551.303, "eval_steps_per_second": 17.244, "train/total_time_seconds": 1861.7832489609718, "train/time_per_step_avg": 0.1672628151997924, "train/epoch_time_elapsed": 3078.170626375824, "train/estimated_remaining_minutes": 0.0}
 {"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785788490.2556565, "train_runtime": 3078.7208, "train_samples_per_second": 207.879, "train_steps_per_second": 3.248, "total_flos": 3.506404589568e+16, "train_loss": 3.273249367904663, "train/total_time_seconds": 1861.7832489609718, "train/time_per_step_avg": 0.1672628151997924, "train/epoch_time_elapsed": 3078.7157305553555, "train/estimated_remaining_minutes": 0.0}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785788509.1161447, "eval_loss": 1.3872374296188354, "eval_runtime": 18.857, "eval_samples_per_second": 535.769, "eval_steps_per_second": 16.758, "train/total_time_seconds": 1861.7832489609718, "train/time_per_step_avg": 0.1672628151997924, "train/epoch_time_elapsed": 3097.576219931245, "train/estimated_remaining_minutes": 0.0}

out/relu_run/training_log.jsonl

@@ -0,0 +1,17 @@
+{"step": 10, "epoch": 0.0006370847004109196, "timestamp": 1785788515.9649696, "loss": 17.978007507324218, "grad_norm": 4.431342124938965, "learning_rate": 0.0003, "train/total_time_seconds": 2.8141015209257603, "train/time_per_step_avg": 0.281410152092576, "train/epoch_time_elapsed": 3.3764405250549316, "train/estimated_remaining_minutes": 46.85479032341391, "train/global/act/norm": 26633.040923049444, "train/global/act/mean": -0.0018835791502239607, "train/global/act/std": 0.49934913757814614, "train/global/act/max_abs": 10.493465423583984, "train/global/act/frac_near_dtype_limit": 0.0, "train/global/act/frac_near_user_limit": 0.0, "train/global/grad/norm": 5.841995672470947, "train/global/grad/mean": -5.670259813123616e-07, "train/global/grad/std": 0.0007061814772349122, "train/global/grad/max_abs": 0.150979682803154, "train/global/grad/frac_near_dtype_limit": 0.0, "train/global/grad/frac_near_user_limit": 0.0, "train/global/param/norm": 149.62604269761985, "train/global/param/mean": 0.00025426153845833346, "train/global/param/std": 0.02557734576677353, "train/global/param/max_abs": 1.0003000497817993, "train/global/param/frac_near_dtype_limit": 0.0, "train/global/param/frac_near_user_limit": 0.0, "train/layer_model_layers_4/act/norm": 7603.969414304202, "train/layer_model_layers_4/act/mean": -0.0033013847871468617, "train/layer_model_layers_4/act/std": 0.514794220449786, "train/layer_model_layers_4/act/max_abs": 5.163461208343506, "train/layer_model_layers_4/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_4/act/frac_near_user_limit": 0.0, "train/layer_model_layers_4/grad/norm": 1.1198474073260207, "train/layer_model_layers_4/grad/mean": 7.722473769337734e-07, "train/layer_model_layers_4/grad/std": 0.0005303530694760626, "train/layer_model_layers_4/grad/max_abs": 0.0111083984375, "train/layer_model_layers_4/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_4/grad/frac_near_user_limit": 0.0, "train/layer_model_layers_6/act/norm": 7700.8296946819855, "train/layer_model_layers_6/act/mean": -0.005697576830593439, "train/layer_model_layers_6/act/std": 0.5213566812450333, "train/layer_model_layers_6/act/max_abs": 5.09547758102417, "train/layer_model_layers_6/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_6/act/frac_near_user_limit": 0.0, "train/layer_model_layers_6/grad/norm": 0.9289129086759037, "train/layer_model_layers_6/grad/mean": -7.251062577513816e-07, "train/layer_model_layers_6/grad/std": 0.00043992748955484003, "train/layer_model_layers_6/grad/max_abs": 0.00909423828125, "train/layer_model_layers_6/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_6/grad/frac_near_user_limit": 0.0, "train/layer_model_layers_7/act/norm": 7753.630630796995, "train/layer_model_layers_7/act/mean": -0.0023063072003424168, "train/layer_model_layers_7/act/std": 0.5250076617979635, "train/layer_model_layers_7/act/max_abs": 5.2592573165893555, "train/layer_model_layers_7/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_7/act/frac_near_user_limit": 0.0, "train/layer_model_layers_7/grad/norm": 0.8269160169617942, "train/layer_model_layers_7/grad/mean": 5.167229144657616e-07, "train/layer_model_layers_7/grad/std": 0.00039162257068889306, "train/layer_model_layers_7/grad/max_abs": 0.00811767578125, "train/layer_model_layers_7/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_7/grad/frac_near_user_limit": 0.0, "train/layer__model_layers_3/param/norm": 43.776635379143286, "train/layer__model_layers_3/param/mean": 0.0004515971410480535, "train/layer__model_layers_3/param/std": 0.02931648996637393, "train/layer__model_layers_3/param/max_abs": 1.0003000497817993, "train/layer__model_layers_3/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_3/param/frac_near_user_limit": 0.0, "train/layer__model_layers_4/param/norm": 43.75929275597046, "train/layer__model_layers_4/param/mean": 0.00044376830656280795, "train/layer__model_layers_4/param/std": 0.029304992986155482, "train/layer__model_layers_4/param/max_abs": 1.0003000497817993, "train/layer__model_layers_4/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_4/param/frac_near_user_limit": 0.0, "train/layer__model_layers_5/param/norm": 43.7580416053065, "train/layer__model_layers_5/param/mean": 0.00044606021379980953, "train/layer__model_layers_5/param/std": 0.029304119692604123, "train/layer__model_layers_5/param/max_abs": 1.0003000497817993, "train/layer__model_layers_5/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_5/param/frac_near_user_limit": 0.0, "train/layer_model_layers_0/act/norm": 7382.435077780564, "train/layer_model_layers_0/act/mean": -0.0017857378742729242, "train/layer_model_layers_0/act/std": 0.49984092685026743, "train/layer_model_layers_0/act/max_abs": 5.179662704467773, "train/layer_model_layers_0/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_0/act/frac_near_user_limit": 0.0, "train/layer_model_layers_0/grad/norm": 3.4798538923554747, "train/layer_model_layers_0/grad/mean": -2.0058703380074445e-06, "train/layer_model_layers_0/grad/std": 0.0016480384182711357, "train/layer_model_layers_0/grad/max_abs": 0.0498046875, "train/layer_model_layers_0/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_0/grad/frac_near_user_limit": 0.0, "train/layer_model_layers_1/act/norm": 7452.829880909455, "train/layer_model_layers_1/act/mean": -0.005003037092347558, "train/layer_model_layers_1/act/std": 0.5045703622971124, "train/layer_model_layers_1/act/max_abs": 5.1179094314575195, "train/layer_model_layers_1/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_1/act/frac_near_user_limit": 0.0, "train/layer_model_layers_1/grad/norm": 2.4285759175894723, "train/layer_model_layers_1/grad/mean": -1.099087455100137e-06, "train/layer_model_layers_1/grad/std": 0.0011501597996885437, "train/layer_model_layers_1/grad/max_abs": 0.023193359375, "train/layer_model_layers_1/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_1/grad/frac_near_user_limit": 0.0, "train/layer__model_layers_1/param/norm": 43.770746824291166, "train/layer__model_layers_1/param/mean": 0.0004633841025983152, "train/layer__model_layers_1/param/std": 0.029312363459844698, "train/layer__model_layers_1/param/max_abs": 1.0003000497817993, "train/layer__model_layers_1/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_1/param/frac_near_user_limit": 0.0, "train/layer__model_layers_7/param/norm": 43.77243262093447, "train/layer__model_layers_7/param/mean": 0.00045408718253720724, "train/layer__model_layers_7/param/std": 0.029313637561711063, "train/layer__model_layers_7/param/max_abs": 1.0002999305725098, "train/layer__model_layers_7/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_7/param/frac_near_user_limit": 0.0, "train/layer__model_layers_0/param/norm": 43.77034664254232, "train/layer__model_layers_0/param/mean": 0.0004527786048289626, "train/layer__model_layers_0/param/std": 0.02931226007209156, "train/layer__model_layers_0/param/max_abs": 1.0003000497817993, "train/layer__model_layers_0/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_0/param/frac_near_user_limit": 0.0, "train/layer__model_layers_2/param/norm": 43.78590475658179, "train/layer__model_layers_2/param/mean": 0.0004587775780457865, "train/layer__model_layers_2/param/std": 0.02932258776529364, "train/layer__model_layers_2/param/max_abs": 1.0003000497817993, "train/layer__model_layers_2/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_2/param/frac_near_user_limit": 0.0, "train/layer_model_layers_2/act/norm": 7523.942660037624, "train/layer_model_layers_2/act/mean": -0.0033676969962051283, "train/layer_model_layers_2/act/std": 0.5095377658187711, "train/layer_model_layers_2/act/max_abs": 5.506824493408203, "train/layer_model_layers_2/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_2/act/frac_near_user_limit": 0.0, "train/layer_model_layers_2/grad/norm": 1.7407618548286068, "train/layer_model_layers_2/grad/mean": 5.225122429217828e-07, "train/layer_model_layers_2/grad/std": 0.0008244151597935217, "train/layer_model_layers_2/grad/max_abs": 0.0206298828125, "train/layer_model_layers_2/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_2/grad/frac_near_user_limit": 0.0, "train/layer__model_layers_6/param/norm": 43.76839645545982, "train/layer__model_layers_6/param/mean": 0.000444383614933564, "train/layer__model_layers_6/param/std": 0.029311081368219787, "train/layer__model_layers_6/param/max_abs": 1.0003000497817993, "train/layer__model_layers_6/param/frac_near_dtype_limit": 0.0, "train/layer__model_layers_6/param/frac_near_user_limit": 0.0, "train/layer_model_layers_3/act/norm": 7553.646839516973, "train/layer_model_layers_3/act/mean": -0.003920144741781629, "train/layer_model_layers_3/act/std": 0.5113627141279937, "train/layer_model_layers_3/act/max_abs": 5.454132556915283, "train/layer_model_layers_3/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_3/act/frac_near_user_limit": 0.0, "train/layer_model_layers_3/grad/norm": 1.3282225154457479, "train/layer_model_layers_3/grad/mean": -2.4920794195624756e-07, "train/layer_model_layers_3/grad/std": 0.0006290389004447716, "train/layer_model_layers_3/grad/max_abs": 0.0166015625, "train/layer_model_layers_3/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_3/grad/frac_near_user_limit": 0.0, "train/layer_model_layers_5/act/norm": 7650.762571755275, "train/layer_model_layers_5/act/mean": -0.002426301809744193, "train/layer_model_layers_5/act/std": 0.5179811027146253, "train/layer_model_layers_5/act/max_abs": 5.187224864959717, "train/layer_model_layers_5/act/frac_near_dtype_limit": 0.0, "train/layer_model_layers_5/act/frac_near_user_limit": 0.0, "train/layer_model_layers_5/grad/norm": 1.0139853079935386, "train/layer_model_layers_5/grad/mean": -2.3170663562364688e-07, "train/layer_model_layers_5/grad/std": 0.0004802178554256911, "train/layer_model_layers_5/grad/max_abs": 0.00885009765625, "train/layer_model_layers_5/grad/frac_near_dtype_limit": 0.0, "train/layer_model_layers_5/grad/frac_near_user_limit": 0.0}
+{"step": 20, "epoch": 0.0012741694008218392, "timestamp": 1785788518.3405678, "loss": 14.234909057617188, "grad_norm": 2.615661144256592, "learning_rate": 0.0003, "train/total_time_seconds": 4.714085515588522, "train/time_per_step_avg": 0.2357042757794261, "train/epoch_time_elapsed": 5.752054002135992, "train/estimated_remaining_minutes": 39.20547787131121}
+{"step": 30, "epoch": 0.001911254101232759, "timestamp": 1785788520.6741276, "loss": 12.283504486083984, "grad_norm": 1.1047611236572266, "learning_rate": 0.0003, "train/total_time_seconds": 6.56831831485033, "train/time_per_step_avg": 0.21894394382834434, "train/epoch_time_elapsed": 8.085613891482353, "train/estimated_remaining_minutes": 36.381185332809885}
+{"step": 40, "epoch": 0.0025483388016436783, "timestamp": 1785788523.0548508, "loss": 11.60558853149414, "grad_norm": 3.5402185916900635, "learning_rate": 0.0003, "train/total_time_seconds": 8.476060945540667, "train/time_per_step_avg": 0.21190152363851666, "train/epoch_time_elapsed": 10.466337148100138, "train/estimated_remaining_minutes": 35.175652923993766}
+{"step": 50, "epoch": 0.003185423502054598, "timestamp": 1785788525.334902, "loss": 11.039342498779297, "grad_norm": 1.9399423599243164, "learning_rate": 0.0003, "train/total_time_seconds": 10.279786370694637, "train/time_per_step_avg": 0.20559572741389276, "train/epoch_time_elapsed": 12.746387835592031, "train/estimated_remaining_minutes": 34.09462479613722}
+{"step": 60, "epoch": 0.003822508202465518, "timestamp": 1785788527.6648014, "loss": 10.313339996337891, "grad_norm": 1.9820612668991089, "learning_rate": 0.0003, "train/total_time_seconds": 12.12118411809206, "train/time_per_step_avg": 0.20201973530153433, "train/epoch_time_elapsed": 15.076287303119898, "train/estimated_remaining_minutes": 33.46793614828752}
+{"step": 70, "epoch": 0.004459592902876438, "timestamp": 1785788530.0068426, "loss": 9.687300872802734, "grad_norm": 3.337968587875366, "learning_rate": 0.0003, "train/total_time_seconds": 13.992601133883, "train/time_per_step_avg": 0.1998943019126143, "train/epoch_time_elapsed": 17.41832859814167, "train/estimated_remaining_minutes": 33.082506966537665}
+{"step": 80, "epoch": 0.005096677603287357, "timestamp": 1785788532.3824563, "loss": 9.177151489257813, "grad_norm": 1.5134049654006958, "learning_rate": 0.0003, "train/total_time_seconds": 15.89501479268074, "train/time_per_step_avg": 0.19868768490850924, "train/epoch_time_elapsed": 19.793942242860794, "train/estimated_remaining_minutes": 32.849697238206865}
+{"step": 90, "epoch": 0.0057337623036982765, "timestamp": 1785788534.6889303, "loss": 8.634540557861328, "grad_norm": 2.3670191764831543, "learning_rate": 0.0003, "train/total_time_seconds": 17.72838632017374, "train/time_per_step_avg": 0.19698207022415268, "train/epoch_time_elapsed": 22.100416269153357, "train/estimated_remaining_minutes": 32.53487193202255}
+{"step": 100, "epoch": 0.006370847004109196, "timestamp": 1785788537.0360637, "loss": 8.206291961669923, "grad_norm": 1.7685469388961792, "learning_rate": 0.0003, "train/total_time_seconds": 19.59542879834771, "train/time_per_step_avg": 0.1959542879834771, "train/epoch_time_elapsed": 24.447549298405647, "train/estimated_remaining_minutes": 32.332457517273724}
+{"step": 110, "epoch": 0.007007931704520116, "timestamp": 1785788539.357573, "loss": 7.930241394042969, "grad_norm": 1.936025619506836, "learning_rate": 0.0003, "train/total_time_seconds": 21.443528477102518, "train/time_per_step_avg": 0.1862942695617676, "train/epoch_time_elapsed": 26.76905919611454, "train/estimated_remaining_minutes": 32.1328025209915}
+{"step": 120, "epoch": 0.007645016404931036, "timestamp": 1785788541.6890316, "loss": 7.6347198486328125, "grad_norm": 1.6677316427230835, "learning_rate": 0.0003, "train/total_time_seconds": 23.30014830827713, "train/time_per_step_avg": 0.18586062792688607, "train/epoch_time_elapsed": 29.100517459213734, "train/estimated_remaining_minutes": 31.972981289691397}
+{"step": 130, "epoch": 0.008282101105341956, "timestamp": 1785788544.0427034, "loss": 7.354776000976562, "grad_norm": 1.787825107574463, "learning_rate": 0.0003, "train/total_time_seconds": 25.174482189118862, "train/time_per_step_avg": 0.18606163874268533, "train/epoch_time_elapsed": 31.454188734292984, "train/estimated_remaining_minutes": 31.855402462385022}
+{"step": 140, "epoch": 0.008919185805752875, "timestamp": 1785788546.3134768, "loss": 7.153055572509766, "grad_norm": 1.9972710609436035, "learning_rate": 0.0003, "train/total_time_seconds": 26.969181191176176, "train/time_per_step_avg": 0.18493120245635508, "train/epoch_time_elapsed": 33.724962655454874, "train/estimated_remaining_minutes": 31.656681731547273}
+{"step": 150, "epoch": 0.009556270506163795, "timestamp": 1785788548.6791255, "loss": 6.991454315185547, "grad_norm": 1.8723620176315308, "learning_rate": 0.0003, "train/total_time_seconds": 28.858658991754055, "train/time_per_step_avg": 0.18578872621059417, "train/epoch_time_elapsed": 36.090611290186644, "train/estimated_remaining_minutes": 31.58419900764194}
+{"step": 160, "epoch": 0.010193355206574713, "timestamp": 1785788551.0517056, "loss": 6.812380981445313, "grad_norm": 2.079190492630005, "learning_rate": 0.0003, "train/total_time_seconds": 30.757146563380957, "train/time_per_step_avg": 0.18635962445288898, "train/epoch_time_elapsed": 38.46319080144167, "train/estimated_remaining_minutes": 31.52607522746548}
+{"step": 170, "epoch": 0.010830439906985633, "timestamp": 1785788553.3985672, "loss": 6.637260437011719, "grad_norm": 1.8830350637435913, "learning_rate": 0.0003, "train/total_time_seconds": 32.6256785877049, "train/time_per_step_avg": 0.18633077453821897, "train/epoch_time_elapsed": 40.81005243957043, "train/estimated_remaining_minutes": 31.442198089915603}

wandb/debug-internal.log

@@ -710,3 +710,13 @@
 {"time":"2026-08-03T20:20:43.211771656Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2061,"history_lines":6,"events_offset":798,"events_lines":2,"console_offset":2240,"console_lines":6}
 {"time":"2026-08-03T20:20:43.538846331Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-03T20:20:58.211907203Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2067,"history_lines":8,"events_offset":800,"events_lines":2,"console_offset":2246,"console_lines":8}
+{"time":"2026-08-03T20:21:53.106684577Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:21:53.106873714Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-08-03T20:21:53.269828764Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:21:53.27014919Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2075,"history_lines":9,"events_offset":802,"events_lines":6,"console_offset":2254,"console_lines":46}
+{"time":"2026-08-03T20:21:53.414626382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:21:53.865999246Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":2300,"console_lines":1}
+{"time":"2026-08-03T20:21:53.991110684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:22:08.866452214Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2084,"history_lines":6,"events_offset":808,"events_lines":2,"console_offset":2300,"console_lines":7}
+{"time":"2026-08-03T20:22:14.599741707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:22:23.86652555Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2090,"history_lines":6,"events_offset":810,"events_lines":2,"console_offset":2300,"console_lines":1}

wandb/debug.log

@@ -24,3 +24,6 @@ config: {'_wandb': {}}
 2026-08-03 19:30:11,538 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 8, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'gelu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/gelu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 10, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 250, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-baseline-gelu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-03 19:30:11,539 INFO    MainThread:2683529 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 34218496 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee565bcfd0>>
 2026-08-03 19:30:11,539 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 34218496 None
+2026-08-03 20:21:52,586 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 8, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'relu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/relu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 10, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 250, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-baseline-relu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-03 20:21:52,588 INFO    MainThread:2683529 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 34218496 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee565bcfd0>>
+2026-08-03 20:21:52,588 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 34218496 None

wandb/run-20260803_184040-agk6ypdy/files/output.log

@@ -2264,4 +2264,53 @@ Writing model shards: 100%|█████████████████
 Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  6.29it/s]
 100%|████████████████████████████████████| 10000/10000 [51:18<00:00,  3.25it/s]
 {'train_runtime': '3079', 'train_samples_per_second': '207.9', 'train_steps_per_second': '3.248', 'train_loss': '3.273', 'epoch': '0.6371', 'train/total_time_seconds': '1862', 'train/time_per_step_avg': '0.1673', 'train/epoch_time_elapsed': '3079', 'train/estimated_remaining_minutes': '0'}
- 23%|█████████▍                               | 73/316 [00:04<00:14, 17.08it/s]
+100%|████████████████████████████████████████| 316/316 [00:18<00:00, 16.81it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  7.98it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  6.89it/s]
+Found 8 files to upload
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files ✓
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files ✓
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files ✓
+  Committing  ████████████████████  8 / 8 ✓
+
+============================================================
+>>> Activation: relu | Out: out/relu_run
+============================================================
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+  0%|                                                | 0/10000 [00:00<?, ?it/s]/mnt/data/zainulabideen/zain-exp/notebooks/Activation/exp.py:416: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at /pytorch/aten/src/ATen/native/ReduceOps.cpp:1831.)
+  "std": tensor.std().item(),
+  2%|▌                                     | 154/10000 [00:37<39:48,  4.12it/s]
+{'loss': '17.98', 'grad_norm': '4.431', 'learning_rate': '0.0003', 'epoch': '0.0006371', 'train/total_time_seconds': '2.814', 'train/time_per_step_avg': '0.2814', 'train/epoch_time_elapsed': '3.376', 'train/estimated_remaining_minutes': '46.85', 'train/global/act/norm': '2.663e+04', 'train/global/act/mean': '-0.001884', 'train/global/act/std': '0.4993', 'train/global/act/max_abs': '10.49', 'train/global/act/frac_near_dtype_limit': '0', 'train/global/act/frac_near_user_limit': '0', 'train/global/grad/norm': '5.842', 'train/global/grad/mean': '-5.67e-07', 'train/global/grad/std': '0.0007062', 'train/global/grad/max_abs': '0.151', 'train/global/grad/frac_near_dtype_limit': '0', 'train/global/grad/frac_near_user_limit': '0', 'train/global/param/norm': '149.6', 'train/global/param/mean': '0.0002543', 'train/global/param/std': '0.02558', 'train/global/param/max_abs': '1', 'train/global/param/frac_near_dtype_limit': '0', 'train/global/param/frac_near_user_limit': '0', 'train/layer_model_layers_4/act/norm': '7604', 'train/layer_model_layers_4/act/mean': '-0.003301', 'train/layer_model_layers_4/act/std': '0.5148', 'train/layer_model_layers_4/act/max_abs': '5.163', 'train/layer_model_layers_4/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_4/act/frac_near_user_limit': '0', 'train/layer_model_layers_4/grad/norm': '1.12', 'train/layer_model_layers_4/grad/mean': '7.722e-07', 'train/layer_model_layers_4/grad/std': '0.0005304', 'train/layer_model_layers_4/grad/max_abs': '0.01111', 'train/layer_model_layers_4/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_4/grad/frac_near_user_limit': '0', 'train/layer_model_layers_6/act/norm': '7701', 'train/layer_model_layers_6/act/mean': '-0.005698', 'train/layer_model_layers_6/act/std': '0.5214', 'train/layer_model_layers_6/act/max_abs': '5.095', 'train/layer_model_layers_6/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_6/act/frac_near_user_limit': '0', 'train/layer_model_layers_6/grad/norm': '0.9289', 'train/layer_model_layers_6/grad/mean': '-7.251e-07', 'train/layer_model_layers_6/grad/std': '0.0004399', 'train/layer_model_layers_6/grad/max_abs': '0.009094', 'train/layer_model_layers_6/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_6/grad/frac_near_user_limit': '0', 'train/layer_model_layers_7/act/norm': '7754', 'train/layer_model_layers_7/act/mean': '-0.002306', 'train/layer_model_layers_7/act/std': '0.525', 'train/layer_model_layers_7/act/max_abs': '5.259', 'train/layer_model_layers_7/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/act/frac_near_user_limit': '0', 'train/layer_model_layers_7/grad/norm': '0.8269', 'train/layer_model_layers_7/grad/mean': '5.167e-07', 'train/layer_model_layers_7/grad/std': '0.0003916', 'train/layer_model_layers_7/grad/max_abs': '0.008118', 'train/layer_model_layers_7/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/grad/frac_near_user_limit': '0', 'train/layer__model_layers_3/param/norm': '43.78', 'train/layer__model_layers_3/param/mean': '0.0004516', 'train/layer__model_layers_3/param/std': '0.02932', 'train/layer__model_layers_3/param/max_abs': '1', 'train/layer__model_layers_3/param/frac_near_dtype_limit': '0', 'train/layer__model_layers_3/param/frac_near_user_limit': '0', 'train/layer__model_layers_4/param/norm': '43.76', 'train/layer__model_layers_4/param/mean': '0.0004438', 'train/layer__model_layers_4/param/std': '0.0293', 'train/layer__model_layers_4/param/max_abs': '1', 'train/layer__model_layers_4/param/frac_near_dtype_limit': '0', 'train/layer__model_layers_4/param/frac_near_user_limit': '0', 'train/layer__model_layers_5/param/norm': '43.76', 'train/layer__model_layers_5/param/mean': '0.0004461', 'train/layer__model_layers_5/param/std': '0.0293', 'train/layer__model_layers_5/param/max_abs': '1', 'train/layer__model_layers_5/param/frac_near_dtype_limit': '0', 'train/layer__model_layers_5/param/frac_near_user_limit': '0', 'train/layer_model_layers_0/act/norm': '7382', 'train/layer_model_layers_0/act/mean': '-0.001786', 'train/layer_model_layers_0/act/std': '0.4998', 'train/layer_mode
+{'loss': '14.23', 'grad_norm': '2.616', 'learning_rate': '0.0003', 'epoch': '0.001274', 'train/total_time_seconds': '4.714', 'train/time_per_step_avg': '0.2357', 'train/epoch_time_elapsed': '5.752', 'train/estimated_remaining_minutes': '39.21'}
+{'loss': '12.28', 'grad_norm': '1.105', 'learning_rate': '0.0003', 'epoch': '0.001911', 'train/total_time_seconds': '6.568', 'train/time_per_step_avg': '0.2189', 'train/epoch_time_elapsed': '8.086', 'train/estimated_remaining_minutes': '36.38'}
+{'loss': '11.61', 'grad_norm': '3.54', 'learning_rate': '0.0003', 'epoch': '0.002548', 'train/total_time_seconds': '8.476', 'train/time_per_step_avg': '0.2119', 'train/epoch_time_elapsed': '10.47', 'train/estimated_remaining_minutes': '35.18'}
+{'loss': '11.04', 'grad_norm': '1.94', 'learning_rate': '0.0003', 'epoch': '0.003185', 'train/total_time_seconds': '10.28', 'train/time_per_step_avg': '0.2056', 'train/epoch_time_elapsed': '12.75', 'train/estimated_remaining_minutes': '34.09'}
+{'loss': '10.31', 'grad_norm': '1.982', 'learning_rate': '0.0003', 'epoch': '0.003823', 'train/total_time_seconds': '12.12', 'train/time_per_step_avg': '0.202', 'train/epoch_time_elapsed': '15.08', 'train/estimated_remaining_minutes': '33.47'}
+{'loss': '9.687', 'grad_norm': '3.338', 'learning_rate': '0.0003', 'epoch': '0.00446', 'train/total_time_seconds': '13.99', 'train/time_per_step_avg': '0.1999', 'train/epoch_time_elapsed': '17.42', 'train/estimated_remaining_minutes': '33.08'}
+{'loss': '9.177', 'grad_norm': '1.513', 'learning_rate': '0.0003', 'epoch': '0.005097', 'train/total_time_seconds': '15.9', 'train/time_per_step_avg': '0.1987', 'train/epoch_time_elapsed': '19.79', 'train/estimated_remaining_minutes': '32.85'}
+{'loss': '8.635', 'grad_norm': '2.367', 'learning_rate': '0.0003', 'epoch': '0.005734', 'train/total_time_seconds': '17.73', 'train/time_per_step_avg': '0.197', 'train/epoch_time_elapsed': '22.1', 'train/estimated_remaining_minutes': '32.53'}
+{'loss': '8.206', 'grad_norm': '1.769', 'learning_rate': '0.0003', 'epoch': '0.006371', 'train/total_time_seconds': '19.6', 'train/time_per_step_avg': '0.196', 'train/epoch_time_elapsed': '24.45', 'train/estimated_remaining_minutes': '32.33'}
+{'loss': '7.93', 'grad_norm': '1.936', 'learning_rate': '0.0003', 'epoch': '0.007008', 'train/total_time_seconds': '21.44', 'train/time_per_step_avg': '0.1863', 'train/epoch_time_elapsed': '26.77', 'train/estimated_remaining_minutes': '32.13'}
+{'loss': '7.635', 'grad_norm': '1.668', 'learning_rate': '0.0003', 'epoch': '0.007645', 'train/total_time_seconds': '23.3', 'train/time_per_step_avg': '0.1859', 'train/epoch_time_elapsed': '29.1', 'train/estimated_remaining_minutes': '31.97'}
+{'loss': '7.355', 'grad_norm': '1.788', 'learning_rate': '0.0003', 'epoch': '0.008282', 'train/total_time_seconds': '25.17', 'train/time_per_step_avg': '0.1861', 'train/epoch_time_elapsed': '31.45', 'train/estimated_remaining_minutes': '31.86'}
+{'loss': '7.153', 'grad_norm': '1.997', 'learning_rate': '0.0003', 'epoch': '0.008919', 'train/total_time_seconds': '26.97', 'train/time_per_step_avg': '0.1849', 'train/epoch_time_elapsed': '33.72', 'train/estimated_remaining_minutes': '31.66'}
+{'loss': '6.991', 'grad_norm': '1.872', 'learning_rate': '0.0003', 'epoch': '0.009556', 'train/total_time_seconds': '28.86', 'train/time_per_step_avg': '0.1858', 'train/epoch_time_elapsed': '36.09', 'train/estimated_remaining_minutes': '31.58'}

wandb/run-20260803_184040-agk6ypdy/logs/debug-internal.log

@@ -710,3 +710,13 @@
 {"time":"2026-08-03T20:20:43.211771656Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2061,"history_lines":6,"events_offset":798,"events_lines":2,"console_offset":2240,"console_lines":6}
 {"time":"2026-08-03T20:20:43.538846331Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-03T20:20:58.211907203Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2067,"history_lines":8,"events_offset":800,"events_lines":2,"console_offset":2246,"console_lines":8}
+{"time":"2026-08-03T20:21:53.106684577Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:21:53.106873714Z","level":"INFO","msg":"filestream: sending request","total_files":0}
+{"time":"2026-08-03T20:21:53.269828764Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:21:53.27014919Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2075,"history_lines":9,"events_offset":802,"events_lines":6,"console_offset":2254,"console_lines":46}
+{"time":"2026-08-03T20:21:53.414626382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:21:53.865999246Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":2300,"console_lines":1}
+{"time":"2026-08-03T20:21:53.991110684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:22:08.866452214Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2084,"history_lines":6,"events_offset":808,"events_lines":2,"console_offset":2300,"console_lines":7}
+{"time":"2026-08-03T20:22:14.599741707Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T20:22:23.86652555Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2090,"history_lines":6,"events_offset":810,"events_lines":2,"console_offset":2300,"console_lines":1}

wandb/run-20260803_184040-agk6ypdy/logs/debug.log

@@ -24,3 +24,6 @@ config: {'_wandb': {}}
 2026-08-03 19:30:11,538 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 8, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'gelu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/gelu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 10, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 250, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-baseline-gelu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-03 19:30:11,539 INFO    MainThread:2683529 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 34218496 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee565bcfd0>>
 2026-08-03 19:30:11,539 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 34218496 None
+2026-08-03 20:21:52,586 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.15.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 32000, 'hidden_size': 512, 'intermediate_size': 768, 'num_hidden_layers': 8, 'num_attention_heads': 8, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, '_name_or_path': '', 'tokenizer_name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0', 'glu_activation': 'relu', 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/relu_run', 'per_device_train_batch_size': 32, 'num_train_epochs': 1, 'max_steps': 10000, 'learning_rate': 0.0003, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 2, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 10, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': None, 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 250, 'eval_delay': 0, 'per_device_eval_batch_size': 32, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/tiny-llama-baseline-relu', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-03 20:21:52,588 INFO    MainThread:2683529 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 34218496 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee565bcfd0>>
+2026-08-03 20:21:52,588 INFO    MainThread:2683529 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 34218496 None

wandb/run-20260803_184040-agk6ypdy/run-agk6ypdy.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:867ab42501167b18ba856113f37de99eac2c0d871fac744847afb8872aa7f0cf
-size 15663104
+oid sha256:fa543c0b211a491e778418fccfaf30054895af51ce7caa10ab6f631562d5d37b
+size 15794176