Upload entire model folder (part 2)

out/situglu_run/config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "glu_activation": "situglu",
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "tiny_llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "transformers_version": "5.15.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

out/situglu_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d6be740f9cc1b610e0112174084685bea88cec7997bdf4d958fed39c31c63f2
+size 208226160

out/situglu_run/tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

out/situglu_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92f273f1d1d299dc4298c6473e9b657dbdcbec6bf1513b5e1880985f4623cc01
+size 4856

out/situglu_run/training_log.jsonl

@@ -213,3 +213,10 @@
 {"step": 9700, "epoch": 0.6179721593985921, "timestamp": 1785801074.200921, "loss": 2.6776422119140624, "grad_norm": 0.6056819558143616, "learning_rate": 0.0003, "train/total_time_seconds": 1748.8080749474466, "train/time_per_step_avg": 0.17927837900817395, "train/epoch_time_elapsed": 2568.593316324055, "train/estimated_remaining_minutes": 0.9014474613131168}
 {"step": 9750, "epoch": 0.6211575829006466, "timestamp": 1785801085.6879976, "loss": 2.673092041015625, "grad_norm": 0.6017354726791382, "learning_rate": 0.0003, "train/total_time_seconds": 1757.7858308330178, "train/time_per_step_avg": 0.17941385079175234, "train/epoch_time_elapsed": 2580.080393217504, "train/estimated_remaining_minutes": 0.7511905259970162}
 {"step": 9800, "epoch": 0.6243430064027012, "timestamp": 1785801097.2578015, "loss": 2.668793029785156, "grad_norm": 0.5871034264564514, "learning_rate": 0.0003, "train/total_time_seconds": 1766.8197576925159, "train/time_per_step_avg": 0.18011682745069266, "train/epoch_time_elapsed": 2591.650197800249, "train/estimated_remaining_minutes": 0.6009591012559578}
+{"step": 9850, "epoch": 0.6275284299047559, "timestamp": 1785801108.7275121, "loss": 2.649367370605469, "grad_norm": 0.5889452695846558, "learning_rate": 0.0003, "train/total_time_seconds": 1775.81880126521, "train/time_per_step_avg": 0.18032970432192086, "train/epoch_time_elapsed": 2603.1199080720544, "train/estimated_remaining_minutes": 0.45071543179320045}
+{"step": 9900, "epoch": 0.6307138534068104, "timestamp": 1785801120.321459, "loss": 2.6719070434570313, "grad_norm": 0.599418580532074, "learning_rate": 0.0003, "train/total_time_seconds": 1784.840480107814, "train/time_per_step_avg": 0.18020722415298224, "train/epoch_time_elapsed": 2614.7138550803065, "train/estimated_remaining_minutes": 0.30047819530434583}
+{"step": 9950, "epoch": 0.633899276908865, "timestamp": 1785801131.8705418, "loss": 2.6650796508789063, "grad_norm": 0.591640293598175, "learning_rate": 0.0003, "train/total_time_seconds": 1793.874307539314, "train/time_per_step_avg": 0.1805550627410412, "train/epoch_time_elapsed": 2626.262937899679, "train/estimated_remaining_minutes": 0.1502407292746494}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785801143.387927, "loss": 2.6560983276367187, "grad_norm": 0.5918380618095398, "learning_rate": 0.0003, "train/total_time_seconds": 1802.8858498744667, "train/time_per_step_avg": 0.18045369766652583, "train/epoch_time_elapsed": 2637.780323486775, "train/estimated_remaining_minutes": 0.0}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785801161.0730057, "eval_loss": 1.3350183963775635, "eval_runtime": 17.6833, "eval_samples_per_second": 571.329, "eval_steps_per_second": 17.87, "train/total_time_seconds": 1802.8858498744667, "train/time_per_step_avg": 0.18045369766652583, "train/epoch_time_elapsed": 2655.4653998985887, "train/estimated_remaining_minutes": 0.0}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785801161.8369784, "train_runtime": 2656.2329, "train_samples_per_second": 240.943, "train_steps_per_second": 3.765, "total_flos": 7.01270851584e+16, "train_loss": 3.201225547790527, "train/total_time_seconds": 1802.8858498744667, "train/time_per_step_avg": 0.18045369766652583, "train/epoch_time_elapsed": 2656.2293715514243, "train/estimated_remaining_minutes": 0.0}
+{"step": 10000, "epoch": 0.6370847004109196, "timestamp": 1785801179.5617316, "eval_loss": 1.3350183963775635, "eval_runtime": 17.7211, "eval_samples_per_second": 570.111, "eval_steps_per_second": 17.832, "train/total_time_seconds": 1802.8858498744667, "train/time_per_step_avg": 0.18045369766652583, "train/epoch_time_elapsed": 2673.954127140343, "train/estimated_remaining_minutes": 0.0}

out/sweep_summary.json

@@ -0,0 +1,27 @@
+[
+  {
+    "activation": "silu",
+    "eval_loss": 1.3351057767868042,
+    "out": "out/silu_run"
+  },
+  {
+    "activation": "gelu",
+    "eval_loss": 1.3375693559646606,
+    "out": "out/gelu_run"
+  },
+  {
+    "activation": "relu",
+    "eval_loss": 1.3447538614273071,
+    "out": "out/relu_run"
+  },
+  {
+    "activation": "bilinear",
+    "eval_loss": 1.3317979574203491,
+    "out": "out/bilinear_run"
+  },
+  {
+    "activation": "situglu",
+    "eval_loss": 1.3350183963775635,
+    "out": "out/situglu_run"
+  }
+]

wandb/debug-internal.log

@@ -2162,3 +2162,14 @@
 {"time":"2026-08-03T23:51:26.978657184Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1098,"history_lines":2,"events_offset":2332,"events_lines":2,"console_offset":1602,"console_lines":1}
 {"time":"2026-08-03T23:51:39.891939127Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-03T23:51:39.892301575Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1100,"history_lines":2,"events_offset":2334,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:51:53.169548755Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:51:53.169830715Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1102,"history_lines":1,"events_offset":2336,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:52:03.778748531Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:52:03.779054315Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1103,"history_lines":1,"events_offset":2338,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:52:17.655389931Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:52:17.655637098Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1104,"history_lines":1,"events_offset":2340,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:52:33.441366437Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:52:33.441637237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1105,"history_lines":1,"events_offset":2342,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:53:01.691230935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:53:01.691568975Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1106,"history_lines":1,"events_offset":2344,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:53:12.326969876Z","level":"ERROR","msg":"monitor: error sampling metrics: process does not exist"}

wandb/run-20260803_185911-6gj59z3n/files/output.log

@@ -1600,7 +1600,7 @@ Writing model shards: 100%|█████████████████
 Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  4.32it/s]
 /mnt/data/zainulabideen/zain-exp/notebooks/2/Activation/exp.py:416: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at /pytorch/aten/src/ATen/native/ReduceOps.cpp:1831.)
   "std": tensor.std().item(),
- 98%|████████████████████████████████████▎| 9823/10000 [43:16<00:40,  4.34it/s]
+100%|████████████████████████████████████| 10000/10000 [44:15<00:00,  4.36it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '2.684', 'grad_norm': '0.6071', 'learning_rate': '0.0003', 'epoch': '0.5766', 'train/total_time_seconds': '1632', 'train/time_per_step_avg': '0.1829', 'train/epoch_time_elapsed': '2402', 'train/estimated_remaining_minutes': '2.854', 'train/global/act/norm': '2.237e+05', 'train/global/act/mean': '-1.247', 'train/global/act/std': '2.912', 'train/global/act/max_abs': '50.88', 'train/global/act/frac_near_dtype_limit': '0', 'train/global/act/frac_near_user_limit': '1.602e-09', 'train/global/grad/norm': '0.5736', 'train/global/grad/mean': '4.523e-09', 'train/global/grad/std': '5.622e-05', 'train/global/grad/max_abs': '0.008503', 'train/global/grad/frac_near_dtype_limit': '0', 'train/global/grad/frac_near_user_limit': '0', 'train/global/param/norm': '312.3', 'train/global/param/mean': '-3.271e-05', 'train/global/param/std': '0.04329', 'train/global/param/max_abs': '1.66', 'train/global/param/frac_near_dtype_limit': '0', 'train/global/param/frac_near_user_limit': '0', 'train/layer__model_layers_14/param/norm': '67.47', 'train/layer__model_layers_14/param/mean': '0.0005312', 'train/layer__model_layers_14/param/std': '0.04519', 'train/layer__model_layers_14/param/max_abs': '1.248', 'train/layer__model_layers_14/param/frac_near_dtype_limit': '0', 'train/layer__model_layers_14/param/frac_near_user_limit': '0', 'train/layer_model_layers_0/act/norm': '1.108e+04', 'train/layer_model_layers_0/act/mean': '-0.03863', 'train/layer_model_layers_0/act/std': '0.7096', 'train/layer_model_layers_0/act/max_abs': '21.88', 'train/layer_model_layers_0/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/act/frac_near_user_limit': '0', 'train/layer_model_layers_0/grad/norm': '0.1818', 'train/layer_model_layers_0/grad/mean': '7.056e-08', 'train/layer_model_layers_0/grad/std': '8.612e-05', 'train/layer_model_layers_0/grad/max_abs': '0.00177', 'train/layer_model_layers_0/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/grad/frac_near_user_limit': '0', 'train/layer_model_layers_7/act/norm': '1.294e+04', 'train/layer_model_layers_7/act/mean': '-0.04034', 'train/layer_model_layers_7/act/std': '0.8289', 'train/layer_model_layers_7/act/max_abs': '42.14', 'train/layer_model_layers_7/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/act/frac_near_user_limit': '0', 'train/layer_model_layers_7/grad/norm': '0.1009', 'train/layer_model_layers_7/grad/mean': '1.158e-08', 'train/layer_model_layers_7/grad/std': '4.778e-05', 'train/layer_model_layers_7/grad/max_abs': '0.00119', 'train/layer_model_layers_7/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/grad/frac_near_user_limit': '0', 'train/layer_model_layers_9/act/norm': '1.571e+04', 'train/layer_model_layers_9/act/mean': '-0.0343', 'train/layer_model_layers_9/act/std': '1.006', 'train/layer_model_layers_9/act/max_abs': '42.15', 'train/layer_model_layers_9/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/act/frac_near_user_limit': '0', 'train/layer_model_layers_9/grad/norm': '0.1263', 'train/layer_model_layers_9/grad/mean': '-4.755e-09', 'train/layer_model_layers_9/grad/std': '5.98e-05', 'train/layer_model_layers_9/grad/max_abs': '0.001755', 'train/layer_model_layers_9/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/grad/frac_near_user_limit': '0', 'train/layer_model_layers_8/act/norm': '1.49e+04', 'train/layer_model_layers_8/act/mean': '-0.04808', 'train/layer_model_layers_8/act/std': '0.9542', 'train/layer_model_layers_8/act/max_abs': '42.83', 'train/layer_model_layers_8/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/act/frac_near_user_limit': '0', 'train/layer_model_layers_8/grad/norm': '0.1243', 'train/layer_model_layers_8/grad/mean': '-5.622e-08', 'train/layer_model_layers_8/grad/std': '5.888e-05', 'train/layer_model_layers_8/grad/max_abs': '0.001465', 'train/layer_model_layers_8/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/grad/frac_near_user_limit': '0', 'train/layer__model_layers_2/param/norm': '51.72', 'train/layer__model_layers_2/param/mean': '0.0003779', 'train/layer__model_layers_2/param/std':
 {'loss': '2.693', 'grad_norm': '0.6083', 'learning_rate': '0.0003', 'epoch': '0.5797', 'train/total_time_seconds': '1641', 'train/time_per_step_avg': '0.1829', 'train/epoch_time_elapsed': '2413', 'train/estimated_remaining_minutes': '2.704'}
 {'loss': '2.689', 'grad_norm': '0.5884', 'learning_rate': '0.0003', 'epoch': '0.5829', 'train/total_time_seconds': '1650', 'train/time_per_step_avg': '0.1801', 'train/epoch_time_elapsed': '2425', 'train/estimated_remaining_minutes': '2.554'}
@@ -1611,7 +1611,7 @@ Writing model shards: 100%|█████████████████
 {'loss': '2.647', 'grad_norm': '0.6001', 'learning_rate': '0.0003', 'epoch': '0.5989', 'train/total_time_seconds': '1695', 'train/time_per_step_avg': '0.1799', 'train/epoch_time_elapsed': '2482', 'train/estimated_remaining_minutes': '1.803'}
 {'loss': '2.685', 'grad_norm': '0.595', 'learning_rate': '0.0003', 'epoch': '0.602', 'train/total_time_seconds': '1704', 'train/time_per_step_avg': '0.18', 'train/epoch_time_elapsed': '2493', 'train/estimated_remaining_minutes': '1.652'}
 {'loss': '2.677', 'grad_norm': '0.6043', 'learning_rate': '0.0003', 'epoch': '0.6052', 'train/total_time_seconds': '1713', 'train/time_per_step_avg': '0.18', 'train/epoch_time_elapsed': '2505', 'train/estimated_remaining_minutes': '1.502'}
-                                                                               
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
 {'eval_loss': '1.342', 'eval_runtime': '17.37', 'eval_samples_per_second': '581.8', 'eval_steps_per_second': '18.2', 'epoch': '0.6052', 'train/total_time_seconds': '1713', 'train/time_per_step_avg': '0.18', 'train/epoch_time_elapsed': '2522', 'train/estimated_remaining_minutes': '1.502'}
 {'loss': '2.672', 'grad_norm': '0.5793', 'learning_rate': '0.0003', 'epoch': '0.6084', 'train/total_time_seconds': '1722', 'train/time_per_step_avg': '0.1829', 'train/epoch_time_elapsed': '2534', 'train/estimated_remaining_minutes': '1.352', 'train/global/act/norm': '2.28e+05', 'train/global/act/mean': '-1.274', 'train/global/act/std': '2.968', 'train/global/act/max_abs': '56.18', 'train/global/act/frac_near_dtype_limit': '0', 'train/global/act/frac_near_user_limit': '3.204e-09', 'train/global/grad/norm': '0.5721', 'train/global/grad/mean': '-2.636e-09', 'train/global/grad/std': '5.607e-05', 'train/global/grad/max_abs': '0.01259', 'train/global/grad/frac_near_dtype_limit': '0', 'train/global/grad/frac_near_user_limit': '0', 'train/global/param/norm': '315.2', 'train/global/param/mean': '-1.677e-06', 'train/global/param/std': '0.04369', 'train/global/param/max_abs': '1.666', 'train/global/param/frac_near_dtype_limit': '0', 'train/global/param/frac_near_user_limit': '0', 'train/layer__model_layers_14/param/norm': '68.14', 'train/layer__model_layers_14/param/mean': '0.0005316', 'train/layer__model_layers_14/param/std': '0.04564', 'train/layer__model_layers_14/param/max_abs': '1.242', 'train/layer__model_layers_14/param/frac_near_dtype_limit': '0', 'train/layer__model_layers_14/param/frac_near_user_limit': '0', 'train/layer_model_layers_0/act/norm': '1.114e+04', 'train/layer_model_layers_0/act/mean': '-0.0399', 'train/layer_model_layers_0/act/std': '0.7132', 'train/layer_model_layers_0/act/max_abs': '20.75', 'train/layer_model_layers_0/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/act/frac_near_user_limit': '0', 'train/layer_model_layers_0/grad/norm': '0.1807', 'train/layer_model_layers_0/grad/mean': '-4.254e-08', 'train/layer_model_layers_0/grad/std': '8.558e-05', 'train/layer_model_layers_0/grad/max_abs': '0.00148', 'train/layer_model_layers_0/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_0/grad/frac_near_user_limit': '0', 'train/layer_model_layers_7/act/norm': '1.309e+04', 'train/layer_model_layers_7/act/mean': '-0.04101', 'train/layer_model_layers_7/act/std': '0.8384', 'train/layer_model_layers_7/act/max_abs': '43.94', 'train/layer_model_layers_7/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/act/frac_near_user_limit': '0', 'train/layer_model_layers_7/grad/norm': '0.09971', 'train/layer_model_layers_7/grad/mean': '-2.583e-08', 'train/layer_model_layers_7/grad/std': '4.722e-05', 'train/layer_model_layers_7/grad/max_abs': '0.001259', 'train/layer_model_layers_7/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_7/grad/frac_near_user_limit': '0', 'train/layer_model_layers_9/act/norm': '1.59e+04', 'train/layer_model_layers_9/act/mean': '-0.03455', 'train/layer_model_layers_9/act/std': '1.019', 'train/layer_model_layers_9/act/max_abs': '45.26', 'train/layer_model_layers_9/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/act/frac_near_user_limit': '0', 'train/layer_model_layers_9/grad/norm': '0.1251', 'train/layer_model_layers_9/grad/mean': '-6.063e-09', 'train/layer_model_layers_9/grad/std': '5.925e-05', 'train/layer_model_layers_9/grad/max_abs': '0.003326', 'train/layer_model_layers_9/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_9/grad/frac_near_user_limit': '0', 'train/layer_model_layers_8/act/norm': '1.511e+04', 'train/layer_model_layers_8/act/mean': '-0.04852', 'train/layer_model_layers_8/act/std': '0.968', 'train/layer_model_layers_8/act/max_abs': '45.29', 'train/layer_model_layers_8/act/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/act/frac_near_user_limit': '0', 'train/layer_model_layers_8/grad/norm': '0.1236', 'train/layer_model_layers_8/grad/mean': '1.829e-09', 'train/layer_model_layers_8/grad/std': '5.856e-05', 'train/layer_model_layers_8/grad/max_abs': '0.001793', 'train/layer_model_layers_8/grad/frac_near_dtype_limit': '0', 'train/layer_model_layers_8/grad/frac_near_user_limit': '0', 'train/layer__model_layers_2/param/norm': '52.32', 'train/layer__model_layers_2/param/mean': '0.0003717', 'train/layer__model_layers_2/param/std
 {'loss': '2.674', 'grad_norm': '0.5856', 'learning_rate': '0.0003', 'epoch': '0.6116', 'train/total_time_seconds': '1731', 'train/time_per_step_avg': '0.1834', 'train/epoch_time_elapsed': '2546', 'train/estimated_remaining_minutes': '1.202'}
@@ -1619,3 +1619,53 @@ Writing model shards: 100%|█████████████████
 {'loss': '2.678', 'grad_norm': '0.6057', 'learning_rate': '0.0003', 'epoch': '0.618', 'train/total_time_seconds': '1749', 'train/time_per_step_avg': '0.1793', 'train/epoch_time_elapsed': '2569', 'train/estimated_remaining_minutes': '0.9014'}
 {'loss': '2.673', 'grad_norm': '0.6017', 'learning_rate': '0.0003', 'epoch': '0.6212', 'train/total_time_seconds': '1758', 'train/time_per_step_avg': '0.1794', 'train/epoch_time_elapsed': '2580', 'train/estimated_remaining_minutes': '0.7512'}
 {'loss': '2.669', 'grad_norm': '0.5871', 'learning_rate': '0.0003', 'epoch': '0.6243', 'train/total_time_seconds': '1767', 'train/time_per_step_avg': '0.1801', 'train/epoch_time_elapsed': '2592', 'train/estimated_remaining_minutes': '0.601'}
+{'loss': '2.649', 'grad_norm': '0.5889', 'learning_rate': '0.0003', 'epoch': '0.6275', 'train/total_time_seconds': '1776', 'train/time_per_step_avg': '0.1803', 'train/epoch_time_elapsed': '2603', 'train/estimated_remaining_minutes': '0.4507'}
+{'loss': '2.672', 'grad_norm': '0.5994', 'learning_rate': '0.0003', 'epoch': '0.6307', 'train/total_time_seconds': '1785', 'train/time_per_step_avg': '0.1802', 'train/epoch_time_elapsed': '2615', 'train/estimated_remaining_minutes': '0.3005'}
+{'loss': '2.665', 'grad_norm': '0.5916', 'learning_rate': '0.0003', 'epoch': '0.6339', 'train/total_time_seconds': '1794', 'train/time_per_step_avg': '0.1806', 'train/epoch_time_elapsed': '2626', 'train/estimated_remaining_minutes': '0.1502'}
+{'loss': '2.656', 'grad_norm': '0.5918', 'learning_rate': '0.0003', 'epoch': '0.6371', 'train/total_time_seconds': '1803', 'train/time_per_step_avg': '0.1805', 'train/epoch_time_elapsed': '2638', 'train/estimated_remaining_minutes': '0'}
+{'eval_loss': '1.335', 'eval_runtime': '17.68', 'eval_samples_per_second': '571.3', 'eval_steps_per_second': '17.87', 'epoch': '0.6371', 'train/total_time_seconds': '1803', 'train/time_per_step_avg': '0.1805', 'train/epoch_time_elapsed': '2655', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  4.32it/s]
+100%|████████████████████████████████████| 10000/10000 [44:16<00:00,  3.76it/s]
+{'train_runtime': '2656', 'train_samples_per_second': '240.9', 'train_steps_per_second': '3.765', 'train_loss': '3.201', 'epoch': '0.6371', 'train/total_time_seconds': '1803', 'train/time_per_step_avg': '0.1805', 'train/epoch_time_elapsed': '2656', 'train/estimated_remaining_minutes': '0'}
+100%|████████████████████████████████████████| 316/316 [00:17<00:00, 17.90it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  3.76it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████████████████| 1/1 [00:00<00:00,  4.98it/s]
+Found 8 files to upload
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files  128MB · 255MB/s
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files  207MB · 226MB/s
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ██████████░░░░░░░░░░  1 / 2 files  208MB · 158MB/s
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files  208MB · 111MB/s ✓
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files  208MB · 77.8MB/s ✓
+  Committing  ░░░░░░░░░░░░░░░░░░░░  0 / 8
+  Preparing   ████████████████████  8 / 8 ✓
+  Uploading   ████████████████████  2 / 2 files  208MB · 77.8MB/s ✓
+  Committing  ████████████████████  8 / 8 ✓
+
+Sweep complete:
+  silu          eval_loss=1.3351
+  gelu          eval_loss=1.3376
+  relu          eval_loss=1.3448
+  bilinear      eval_loss=1.3318
+  situglu       eval_loss=1.3350

wandb/run-20260803_185911-6gj59z3n/logs/debug-core.log

@@ -38,3 +38,7 @@
 {"time":"2026-08-03T18:59:11.744815052Z","level":"INFO","msg":"handleInformInit: received","streamId":"6gj59z3n","id":"6(@)"}
 {"time":"2026-08-03T18:59:11.802212036Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"6gj59z3n","id":"6(@)"}
 {"time":"2026-08-03T18:59:17.391169095Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"hn0whg4hq3rh"}
+{"time":"2026-08-03T23:53:05.979080222Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-03T23:53:05.979079973Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-03T23:53:05.979185799Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-03T23:53:05.979190399Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}

wandb/run-20260803_185911-6gj59z3n/logs/debug-internal.log

@@ -2162,3 +2162,14 @@
 {"time":"2026-08-03T23:51:26.978657184Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1098,"history_lines":2,"events_offset":2332,"events_lines":2,"console_offset":1602,"console_lines":1}
 {"time":"2026-08-03T23:51:39.891939127Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-03T23:51:39.892301575Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1100,"history_lines":2,"events_offset":2334,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:51:53.169548755Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:51:53.169830715Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1102,"history_lines":1,"events_offset":2336,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:52:03.778748531Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:52:03.779054315Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1103,"history_lines":1,"events_offset":2338,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:52:17.655389931Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:52:17.655637098Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1104,"history_lines":1,"events_offset":2340,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:52:33.441366437Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:52:33.441637237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1105,"history_lines":1,"events_offset":2342,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:53:01.691230935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-03T23:53:01.691568975Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1106,"history_lines":1,"events_offset":2344,"events_lines":2,"console_offset":1602,"console_lines":1}
+{"time":"2026-08-03T23:53:12.326969876Z","level":"ERROR","msg":"monitor: error sampling metrics: process does not exist"}

wandb/run-20260803_185911-6gj59z3n/run-6gj59z3n.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec012bae9a267eddf6b2eaada9f3f0febd6380bd3d9d46f154400e36504af769
-size 27983872
+oid sha256:22b67b848efa44867fc69a3df6f8e2718f926dacd157164af41c42cca0b4c0ce
+size 28180480