Auto upload zain 2026-08-14T21:04:09.516498 (part 5)

.gitattributes

@@ -51,3 +51,4 @@ zain/Activation/wandb/run-20260814_202806-g2ch84q3/run-g2ch84q3.wandb filter=lfs
 zain/Activation/wandb/run-20260814_203803-7p5140bq/run-7p5140bq.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260814_203945-j86czemo/run-j86czemo.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/wandb/run-20260814_205135-0vnbmyac/run-0vnbmyac.wandb filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260814_210109-qauftiuh/run-qauftiuh.wandb filter=lfs diff=lfs merge=lfs -text

zain/Activation/wandb/run-20260814_202806-g2ch84q3/logs/debug-core.log

@@ -33,3 +33,10 @@
 {"time":"2026-08-14T20:51:35.100639214Z","level":"INFO","msg":"handleInformInit: received","streamId":"0vnbmyac","id":"1(@)"}
 {"time":"2026-08-14T20:51:35.363188061Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"0vnbmyac","id":"1(@)"}
 {"time":"2026-08-14T20:51:40.739065686Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:54.898587298Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.393521917Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.395053333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.395670668Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.967276961Z","level":"INFO","msg":"handleInformInit: received","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:01:56.228683674Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:02:01.684005979Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"atuum5cul6gx"}

zain/Activation/wandb/run-20260814_203803-7p5140bq/logs/debug-core.log

@@ -42,3 +42,7 @@
 {"time":"2026-08-14T20:54:42.40094641Z","level":"INFO","msg":"connection: closed successfully","id":"2(@)"}
 {"time":"2026-08-14T20:54:42.400864953Z","level":"INFO","msg":"processOutgoingData: finished","id":"2(@)"}
 {"time":"2026-08-14T20:54:42.400960827Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"2(@)"}
+{"time":"2026-08-14T21:01:09.497626422Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"3(@)"}
+{"time":"2026-08-14T21:01:09.714752804Z","level":"INFO","msg":"handleInformInit: received","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:01:09.994191533Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:01:15.507216386Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"wd49dhcig506"}

zain/Activation/wandb/run-20260814_203945-j86czemo/logs/debug-core.log

@@ -33,3 +33,10 @@
 {"time":"2026-08-14T20:51:35.100639214Z","level":"INFO","msg":"handleInformInit: received","streamId":"0vnbmyac","id":"1(@)"}
 {"time":"2026-08-14T20:51:35.363188061Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"0vnbmyac","id":"1(@)"}
 {"time":"2026-08-14T20:51:40.739065686Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:54.898587298Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.393521917Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.395053333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.395670668Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.967276961Z","level":"INFO","msg":"handleInformInit: received","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:01:56.228683674Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:02:01.684005979Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"atuum5cul6gx"}

zain/Activation/wandb/run-20260814_205135-0vnbmyac/files/config.yaml

@@ -0,0 +1,439 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            liwf3fenqdx4bhetb7k4v2cc352a1xkd:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml
+                    - --variants
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                    - mlp-waleed10
+                    - mlp-silu-waleed10
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "696203104256"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T20:51:35.098227Z"
+                writerId: liwf3fenqdx4bhetb7k4v2cc352a1xkd
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: waleed
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 60000
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/A-glu-waleed-9L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 2001280
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 9
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-waleed-9L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 512
+per_device_train_batch_size:
+    value: 512
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-waleed-9L-2.0M-20260814-205134
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0.01

zain/Activation/wandb/run-20260814_205135-0vnbmyac/files/output.log

@@ -68,6 +68,43 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 165
   - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
   - If you are not the owner of the model architecture class, please contact the model code owner to update it.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 158.85it/s]
- 74%|███████▍  | 742/1000 [07:29<02:24,  1.78it/s]?, ?it/s]
+ 80%|████████  | 800/1000 [08:01<01:53,  1.76it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '2.375', 'grad_norm': '0.5391', 'learning_rate': '0.001', 'epoch': '0.3881', 'train/total_time_seconds': '163.7', 'train/time_per_step_avg': '0.1767', 'train/epoch_time_elapsed': '437.3', 'train/estimated_remaining_minutes': '1.061'}
 {'loss': '2.368', 'grad_norm': '0.668', 'learning_rate': '0.001', 'epoch': '0.3989', 'train/total_time_seconds': '167.2', 'train/time_per_step_avg': '0.1767', 'train/epoch_time_elapsed': '448.4', 'train/estimated_remaining_minutes': '0.9792'}
+{'loss': '2.354', 'grad_norm': '0.8398', 'learning_rate': '0.001', 'epoch': '0.4097', 'train/total_time_seconds': '170.8', 'train/time_per_step_avg': '0.1768', 'train/epoch_time_elapsed': '459.5', 'train/estimated_remaining_minutes': '0.8987'}
+{'loss': '2.347', 'grad_norm': '0.6289', 'learning_rate': '0.001', 'epoch': '0.4205', 'train/total_time_seconds': '174.3', 'train/time_per_step_avg': '0.1768', 'train/epoch_time_elapsed': '470.7', 'train/estimated_remaining_minutes': '0.8193'}
+{'loss': '2.323', 'grad_norm': '0.6094', 'learning_rate': '0.001', 'epoch': '0.4313', 'train/total_time_seconds': '177.8', 'train/time_per_step_avg': '0.1769', 'train/epoch_time_elapsed': '481.9', 'train/estimated_remaining_minutes': '0.741'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 90.74it/s]
+ 90%|█████████ | 900/1000 [08:58<00:56,  1.78it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.323', 'grad_norm': '0.9375', 'learning_rate': '0.001', 'epoch': '0.442', 'train/total_time_seconds': '181.4', 'train/time_per_step_avg': '0.177', 'train/epoch_time_elapsed': '493.1', 'train/estimated_remaining_minutes': '0.6636'}
+{'loss': '2.313', 'grad_norm': '0.625', 'learning_rate': '0.001', 'epoch': '0.4528', 'train/total_time_seconds': '184.9', 'train/time_per_step_avg': '0.177', 'train/epoch_time_elapsed': '504.4', 'train/estimated_remaining_minutes': '0.587'}
+{'loss': '2.297', 'grad_norm': '0.6445', 'learning_rate': '0.001', 'epoch': '0.4636', 'train/total_time_seconds': '188.5', 'train/time_per_step_avg': '0.1771', 'train/epoch_time_elapsed': '515.9', 'train/estimated_remaining_minutes': '0.5113'}
+{'loss': '2.289', 'grad_norm': '0.7656', 'learning_rate': '0.001', 'epoch': '0.4744', 'train/total_time_seconds': '192', 'train/time_per_step_avg': '0.1772', 'train/epoch_time_elapsed': '527.1', 'train/estimated_remaining_minutes': '0.4364'}
+{'loss': '2.28', 'grad_norm': '0.6445', 'learning_rate': '0.001', 'epoch': '0.4852', 'train/total_time_seconds': '195.5', 'train/time_per_step_avg': '0.1772', 'train/epoch_time_elapsed': '538.4', 'train/estimated_remaining_minutes': '0.3621'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 165.10it/s]
+100%|██████████| 1000/1000 [10:09<00:00,  1.43i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.268', 'grad_norm': '0.6641', 'learning_rate': '0.001', 'epoch': '0.496', 'train/total_time_seconds': '199.1', 'train/time_per_step_avg': '0.1772', 'train/epoch_time_elapsed': '549.8', 'train/estimated_remaining_minutes': '0.2885'}
+{'loss': '2.269', 'grad_norm': '0.6523', 'learning_rate': '0.001', 'epoch': '0.5067', 'train/total_time_seconds': '202.6', 'train/time_per_step_avg': '0.1773', 'train/epoch_time_elapsed': '561.1', 'train/estimated_remaining_minutes': '0.2156'}
+{'loss': '2.263', 'grad_norm': '0.5156', 'learning_rate': '0.001', 'epoch': '0.5175', 'train/total_time_seconds': '206.2', 'train/time_per_step_avg': '0.1772', 'train/epoch_time_elapsed': '572.5', 'train/estimated_remaining_minutes': '0.1432'}
+{'loss': '2.256', 'grad_norm': '0.625', 'learning_rate': '0.001', 'epoch': '0.5283', 'train/total_time_seconds': '211.8', 'train/time_per_step_avg': '0.1981', 'train/epoch_time_elapsed': '585.9', 'train/estimated_remaining_minutes': '0.07205'}
+{'loss': '2.247', 'grad_norm': '0.5938', 'learning_rate': '0.001', 'epoch': '0.5391', 'train/total_time_seconds': '218.2', 'train/time_per_step_avg': '0.2265', 'train/epoch_time_elapsed': '600', 'train/estimated_remaining_minutes': '0'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.245', 'eval_runtime': '9.564', 'eval_samples_per_second': '996.1', 'eval_steps_per_second': '1.987', 'epoch': '0.5391', 'train/total_time_seconds': '218.2', 'train/time_per_step_avg': '0.2265', 'train/epoch_time_elapsed': '609.6', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 164.88it/s]
+100%|██████████| 1000/1000 [10:09<00:00,  1.64it/s], ?it/s]
+{'train_runtime': '610.4', 'train_samples_per_second': '838.8', 'train_steps_per_second': '1.638', 'train_loss': '2.969', 'epoch': '0.5391', 'train/total_time_seconds': '218.2', 'train/time_per_step_avg': '0.2265', 'train/epoch_time_elapsed': '609.7', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 19/19 [00:08<00:00,  2.19it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 146.08it/s]
+>>> FINISHED glu-waleed-9L successfully

zain/Activation/wandb/run-20260814_205135-0vnbmyac/files/wandb-summary.json

@@ -0,0 +1 @@
+{"total_flos":2.323107545088e+15,"train/grad_norm":0.59375,"train_runtime":610.3715,"train/learning_rate":0.001,"train_samples_per_second":838.833,"eval/loss":2.2454068660736084,"train_loss":2.9690982284545897,"train_steps_per_second":1.638,"_runtime":619,"_step":52,"eval/runtime":9.4741,"train/epoch":0.5390835579514824,"_wandb":{"runtime":619},"train/global_step":1000,"eval/samples_per_second":1005.585,"eval/steps_per_second":2.005,"train/loss":2.2472488403320314,"_timestamp":1.7867413148814697e+09}

zain/Activation/wandb/run-20260814_205135-0vnbmyac/logs/debug-core.log

@@ -33,3 +33,10 @@
 {"time":"2026-08-14T20:51:35.100639214Z","level":"INFO","msg":"handleInformInit: received","streamId":"0vnbmyac","id":"1(@)"}
 {"time":"2026-08-14T20:51:35.363188061Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"0vnbmyac","id":"1(@)"}
 {"time":"2026-08-14T20:51:40.739065686Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:54.898587298Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.393521917Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.395053333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.395670668Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.967276961Z","level":"INFO","msg":"handleInformInit: received","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:01:56.228683674Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:02:01.684005979Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"atuum5cul6gx"}

zain/Activation/wandb/run-20260814_205135-0vnbmyac/logs/debug-internal.log

@@ -67,3 +67,33 @@
 {"time":"2026-08-14T20:58:50.87090741Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-14T20:59:05.742918017Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":2,"events_offset":57,"events_lines":2,"console_offset":62,"console_lines":11}
 {"time":"2026-08-14T20:59:05.878151617Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T20:59:20.742443722Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":1,"events_offset":59,"events_lines":2,"console_offset":70,"console_lines":1}
+{"time":"2026-08-14T20:59:20.870289815Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T20:59:35.742121682Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":38,"history_lines":1,"events_offset":61,"events_lines":2,"console_offset":70,"console_lines":1}
+{"time":"2026-08-14T20:59:35.848508436Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T20:59:50.742723579Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":39,"history_lines":2,"events_offset":63,"events_lines":2,"console_offset":70,"console_lines":1}
+{"time":"2026-08-14T20:59:50.864175424Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:00:05.742280729Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":1,"events_offset":65,"events_lines":2,"console_offset":73,"console_lines":10}
+{"time":"2026-08-14T21:00:05.947551342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:00:20.74290754Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":42,"history_lines":1,"events_offset":67,"events_lines":2,"console_offset":80,"console_lines":1}
+{"time":"2026-08-14T21:00:20.914222267Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:00:35.742834963Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":2,"events_offset":69,"events_lines":2,"console_offset":80,"console_lines":1}
+{"time":"2026-08-14T21:00:35.897885937Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:00:50.742489119Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":45,"history_lines":1,"events_offset":71,"events_lines":2,"console_offset":83,"console_lines":9}
+{"time":"2026-08-14T21:00:50.891763003Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:05.742477262Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":1,"events_offset":73,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:01:05.894426406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:20.743083153Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":75,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:01:20.869218273Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:35.74213026Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":1,"events_offset":77,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:01:35.869483212Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:50.742160362Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":3,"events_offset":79,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-14T21:01:50.870503488Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:55.229430177Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-14T21:01:55.229676237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":52,"history_lines":1,"events_offset":81,"events_lines":1,"console_offset":92,"console_lines":18,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-14T21:01:55.390954146Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:55.392259902Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-14T21:01:55.395081248Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-14T21:01:55.395101308Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-14T21:01:55.395158997Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-14T21:01:55.395162516Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260814_205135-0vnbmyac/logs/debug.log

@@ -17,3 +17,8 @@ config: {'_wandb': {}}
 2026-08-14 20:51:35,738 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 9, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'waleed', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-waleed-9L_run', 'per_device_train_batch_size': 512, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-waleed-9L-2.0M-20260814-205134', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 60000, 'eval_delay': 0, 'per_device_eval_batch_size': 512, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/A-glu-waleed-9L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-14 20:51:35,740 INFO    MainThread:409188 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 2001280 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x153f2878a810>>
 2026-08-14 20:51:35,740 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 2001280 None
+2026-08-14 21:01:54,897 INFO    MainThread:409188 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research1/0vnbmyac
+2026-08-14 21:01:54,898 INFO    MainThread:409188 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-14 21:01:54,898 INFO    MainThread:409188 [wandb_run.py:_restore():2570] restore
+2026-08-14 21:01:54,898 INFO    MainThread:409188 [wandb_run.py:_restore():2576] restore done
+2026-08-14 21:01:55,394 INFO    MainThread:409188 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260814_205135-0vnbmyac/run-0vnbmyac.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7410c90724abdce41213b7563db506d7f43165ece0a1cb0398c28f78659fbef7
-size 327680
+oid sha256:31917ed930323e8621f015f0250e4f312bd76398c56891f1ed3d292127b06bf5
+size 475536

zain/Activation/wandb/run-20260814_210109-qauftiuh/files/output.log

@@ -0,0 +1,54 @@
+[Override] Activated with value = -10.0
+  0%|          | 0/3500 [00:00<?, ?it/s][transformers] `use_return_dict` is deprecated! Use `return_dict` instead!
+[INFO] Causal mask (float with -inf) applied to all attention layers.
+ 74%|███████▍  | 2600/3500 [00:35<00:20, 44.04it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.064', 'grad_norm': '1.579e+07', 'learning_rate': '3.42e-05', 'epoch': '0.1699', 'train/total_time_seconds': '7.486', 'train/time_per_step_avg': '0.3743', 'train/epoch_time_elapsed': '8.92', 'train/estimated_remaining_minutes': '0.04852', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001022', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0001545', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3438', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7285', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1244', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.03015', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4277', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '2.094', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-1.805', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '2.094', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '3.898', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1030', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.002701', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.3555', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '1.875', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-1.875', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.672', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '3.547', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '168.7', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.0002232', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.08252', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.5586', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.5586', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.4609', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.02', 'train/tensor_act_model_layers_0_mlp/norm': '168.7', 'train/tensor_act_model_layers_0_mlp/mean': '0.0002232', 'train/tensor_act_model_layers_0_mlp/std': '0.08252', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.5586', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.5586', 'train/tensor_act_model_layers_0_mlp/max': '0.4609', 'train/tensor_act_model_layers_0_mlp/range': '1.02', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '277.1', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.0003777', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1357', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '
+{'loss': '2.076', 'grad_norm': '1.796e+07', 'learning_rate': '7.02e-05', 'epoch': '0.1712', 'train/total_time_seconds': '13.42', 'train/time_per_step_avg': '0.3356', 'train/epoch_time_elapsed': '15.91', 'train/estimated_remaining_minutes': '0.08456'}
+{'loss': '2.944', 'grad_norm': '9.542e+07', 'learning_rate': '9e-05', 'epoch': '0.1726', 'train/total_time_seconds': '20.16', 'train/time_per_step_avg': '0.3361', 'train/epoch_time_elapsed': '23.77', 'train/estimated_remaining_minutes': '0.1234', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.2', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009842', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0001497', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3691', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7539', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1272', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.02502', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4395', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '2.188', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-1.852', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '2.188', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '4.039', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1061', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.0007019', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.3672', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '1.82', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-1.82', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.703', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '3.523', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '184.3', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.01697', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.08838', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.5781', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.5781', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.5312', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.109', 'train/tensor_act_model_layers_0_mlp/norm': '184.3', 'train/tensor_act_model_layers_0_mlp/mean': '0.01697', 'train/tensor_act_model_layers_0_mlp/std': '0.08838', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.5781', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.5781', 'train/tensor_act_model_layers_0_mlp/max': '0.5312', 'train/tensor_act_model_layers_0_mlp/range': '1.109', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '288', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.01709', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1396', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7305',
+{'loss': '7.066', 'grad_norm': '3.272e+08', 'learning_rate': '9e-05', 'epoch': '0.1739', 'train/total_time_seconds': '25.24', 'train/time_per_step_avg': '0.3155', 'train/epoch_time_elapsed': '29.93', 'train/estimated_remaining_minutes': '0.15'}
+{'loss': '7.805', 'grad_norm': '6.459e+08', 'learning_rate': '9e-05', 'epoch': '0.1753', 'train/total_time_seconds': '30.21', 'train/time_per_step_avg': '0.3021', 'train/epoch_time_elapsed': '35.86', 'train/estimated_remaining_minutes': '0.1743'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 14.07it/s]
+ 77%|███████▋  | 2700/3500 [01:10<04:37,  2.88it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '7.98', 'grad_norm': '1.061e+09', 'learning_rate': '9e-05', 'epoch': '0.1766', 'train/total_time_seconds': '35.79', 'train/time_per_step_avg': '0.2831', 'train/epoch_time_elapsed': '42.72', 'train/estimated_remaining_minutes': '0.2004', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.2', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001068', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0001054', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3438', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3379', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3438', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6816', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1356', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.0166', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4688', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '2.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-1.906', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '2.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '4.25', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1146', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '7.808e-06', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.3965', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '2.047', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-2.047', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.781', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '3.828', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '246.3', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.0437', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.1118', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.6719', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.5586', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.6719', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.23', 'train/tensor_act_model_layers_0_mlp/norm': '246.3', 'train/tensor_act_model_layers_0_mlp/mean': '0.0437', 'train/tensor_act_model_layers_0_mlp/std': '0.1118', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.6719', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.5586', 'train/tensor_act_model_layers_0_mlp/max': '0.6719', 'train/tensor_act_model_layers_0_mlp/range': '1.23', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '334.1', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.04395', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1572', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.75', 'train/te
+{'loss': '8.04', 'grad_norm': '1.552e+09', 'learning_rate': '9e-05', 'epoch': '0.178', 'train/total_time_seconds': '40.77', 'train/time_per_step_avg': '0.2735', 'train/epoch_time_elapsed': '48.72', 'train/estimated_remaining_minutes': '0.2214'}
+{'loss': '8.074', 'grad_norm': '2.106e+09', 'learning_rate': '9e-05', 'epoch': '0.1793', 'train/total_time_seconds': '47.57', 'train/time_per_step_avg': '0.2741', 'train/epoch_time_elapsed': '56.84', 'train/estimated_remaining_minutes': '0.2504', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.5', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001114', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.9', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.000144', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3711', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3555', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3711', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7266', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1366', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.01392', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4707', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '2.328', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-1.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '2.328', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '4.203', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1160', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.001007', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4004', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '2.125', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-2.125', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.867', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '3.992', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '260.2', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.04858', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.1172', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.6602', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6055', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.6602', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.266', 'train/tensor_act_model_layers_0_mlp/norm': '260.2', 'train/tensor_act_model_layers_0_mlp/mean': '0.04858', 'train/tensor_act_model_layers_0_mlp/std': '0.1172', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.6602', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6055', 'train/tensor_act_model_layers_0_mlp/max': '0.6602', 'train/tensor_act_model_layers_0_mlp/range': '1.266', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '345.4', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.04883', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1611', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7422', '
+{'loss': '8.083', 'grad_norm': '2.718e+09', 'learning_rate': '9e-05', 'epoch': '0.1807', 'train/total_time_seconds': '53.66', 'train/time_per_step_avg': '0.2841', 'train/epoch_time_elapsed': '63.98', 'train/estimated_remaining_minutes': '0.2736'}
+{'loss': '8.099', 'grad_norm': '3.355e+09', 'learning_rate': '9e-05', 'epoch': '0.182', 'train/total_time_seconds': '59.49', 'train/time_per_step_avg': '0.2928', 'train/epoch_time_elapsed': '70.81', 'train/estimated_remaining_minutes': '0.2938'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 16.79it/s]
+ 80%|████████  | 2800/3500 [01:47<03:55,  2.97it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.107', 'grad_norm': '4.027e+09', 'learning_rate': '9e-05', 'epoch': '0.1834', 'train/total_time_seconds': '66.04', 'train/time_per_step_avg': '0.3025', 'train/epoch_time_elapsed': '78.87', 'train/estimated_remaining_minutes': '0.3156', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.4', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.00106', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.7', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0001612', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3535', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3535', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3398', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6934', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1374', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.01227', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4746', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '2.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-1.945', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '2.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '4.289', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1172', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.001709', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4043', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '2.016', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-2.016', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.977', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '3.992', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '280.7', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.05615', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.125', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.7188', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6055', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.7188', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.324', 'train/tensor_act_model_layers_0_mlp/norm': '280.7', 'train/tensor_act_model_layers_0_mlp/mean': '0.05615', 'train/tensor_act_model_layers_0_mlp/std': '0.125', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.7188', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6055', 'train/tensor_act_model_layers_0_mlp/max': '0.7188', 'train/tensor_act_model_layers_0_mlp/range': '1.324', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '361', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.0564', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.167', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7578', 'train/
+{'loss': '8.107', 'grad_norm': '4.698e+09', 'learning_rate': '9e-05', 'epoch': '0.1847', 'train/total_time_seconds': '71.83', 'train/time_per_step_avg': '0.3106', 'train/epoch_time_elapsed': '85.68', 'train/estimated_remaining_minutes': '0.3321'}
+{'loss': '8.106', 'grad_norm': '5.402e+09', 'learning_rate': '9e-05', 'epoch': '0.186', 'train/total_time_seconds': '79.07', 'train/time_per_step_avg': '0.315', 'train/epoch_time_elapsed': '94.11', 'train/estimated_remaining_minutes': '0.3533', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.4', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009995', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0001411', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3887', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3887', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3477', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7363', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1385', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.01166', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4785', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '2.672', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-2.672', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '2.406', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '5.078', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1187', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.002106', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4102', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '2.797', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-2.719', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '2.797', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '5.516', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '339.3', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.07568', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.1475', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.7539', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.5625', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.7539', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.316', 'train/tensor_act_model_layers_0_mlp/norm': '339.3', 'train/tensor_act_model_layers_0_mlp/mean': '0.07568', 'train/tensor_act_model_layers_0_mlp/std': '0.1475', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.7539', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.5625', 'train/tensor_act_model_layers_0_mlp/max': '0.7539', 'train/tensor_act_model_layers_0_mlp/range': '1.316', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '408.3', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.07568', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1846', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.8672', '
+{'loss': '8.111', 'grad_norm': '6.073e+09', 'learning_rate': '9e-05', 'epoch': '0.1874', 'train/total_time_seconds': '84.94', 'train/time_per_step_avg': '0.3128', 'train/epoch_time_elapsed': '101', 'train/estimated_remaining_minutes': '0.3666'}
+{'loss': '8.118', 'grad_norm': '6.778e+09', 'learning_rate': '9e-05', 'epoch': '0.1887', 'train/total_time_seconds': '90.9', 'train/time_per_step_avg': '0.3141', 'train/epoch_time_elapsed': '107.9', 'train/estimated_remaining_minutes': '0.3788'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.50it/s]
+ 83%|████████▎ | 2900/3500 [02:24<03:09,  3.16it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.115', 'grad_norm': '7.449e+09', 'learning_rate': '9e-05', 'epoch': '0.1901', 'train/total_time_seconds': '97.56', 'train/time_per_step_avg': '0.3152', 'train/epoch_time_elapsed': '115.9', 'train/estimated_remaining_minutes': '0.3921', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001053', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '4.625e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3789', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3789', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3711', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.75', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1399', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.01068', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4824', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '3.359', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-3.359', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '3.047', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '6.406', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1204', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.003418', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.416', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '3.469', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-3.406', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '3.469', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '6.875', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '461.3', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.1118', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.1953', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '1.062', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.5703', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '1.062', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.633', 'train/tensor_act_model_layers_0_mlp/norm': '461.3', 'train/tensor_act_model_layers_0_mlp/mean': '0.1118', 'train/tensor_act_model_layers_0_mlp/std': '0.1953', 'train/tensor_act_model_layers_0_mlp/max_abs': '1.062', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.5703', 'train/tensor_act_model_layers_0_mlp/max': '1.062', 'train/tensor_act_model_layers_0_mlp/range': '1.633', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '514.4', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.1118', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.2246', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '1.117', 'train/tens
+{'loss': '8.119', 'grad_norm': '8.12e+09', 'learning_rate': '9e-05', 'epoch': '0.1914', 'train/total_time_seconds': '103.5', 'train/time_per_step_avg': '0.3168', 'train/epoch_time_elapsed': '122.8', 'train/estimated_remaining_minutes': '0.4009'}
+{'loss': '8.11', 'grad_norm': '8.791e+09', 'learning_rate': '9e-05', 'epoch': '0.1928', 'train/total_time_seconds': '110.3', 'train/time_per_step_avg': '0.3122', 'train/epoch_time_elapsed': '130.9', 'train/estimated_remaining_minutes': '0.4113', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.6', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001228', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0002613', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3613', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7461', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1412', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.009766', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4883', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '3.938', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-3.938', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '3.688', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '7.625', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1219', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.003174', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4199', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '4.188', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-3.984', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '4.188', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '8.172', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '624.4', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.1582', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.2598', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '1.297', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.5469', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '1.297', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.844', 'train/tensor_act_model_layers_0_mlp/norm': '624.4', 'train/tensor_act_model_layers_0_mlp/mean': '0.1582', 'train/tensor_act_model_layers_0_mlp/std': '0.2598', 'train/tensor_act_model_layers_0_mlp/max_abs': '1.297', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.5469', 'train/tensor_act_model_layers_0_mlp/max': '1.297', 'train/tensor_act_model_layers_0_mlp/range': '1.844', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '665.2', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.1582', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.2832', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '1.438', 'train/tens
+{'loss': '8.114', 'grad_norm': '9.395e+09', 'learning_rate': '9e-05', 'epoch': '0.1941', 'train/total_time_seconds': '116.2', 'train/time_per_step_avg': '0.313', 'train/epoch_time_elapsed': '137.8', 'train/estimated_remaining_minutes': '0.4171'}
+{'loss': '8.117', 'grad_norm': '1.007e+10', 'learning_rate': '9e-05', 'epoch': '0.1955', 'train/total_time_seconds': '122.1', 'train/time_per_step_avg': '0.3117', 'train/epoch_time_elapsed': '144.6', 'train/estimated_remaining_minutes': '0.4209'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 16.62it/s]
+ 86%|████████▌ | 3000/3500 [03:01<02:50,  2.93it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.109', 'grad_norm': '1.067e+10', 'learning_rate': '9e-05', 'epoch': '0.1968', 'train/total_time_seconds': '128.5', 'train/time_per_step_avg': '0.3096', 'train/epoch_time_elapsed': '152.6', 'train/estimated_remaining_minutes': '0.4255', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001099', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0002747', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3711', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3301', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3711', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7012', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1422', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.01007', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.4902', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.25', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.844', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1232', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.003799', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4258', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '4.625', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-4.5', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '4.625', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '9.125', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '803.8', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.209', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.332', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '1.594', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.5664', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '1.594', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '2.16', 'train/tensor_act_model_layers_0_mlp/norm': '803.8', 'train/tensor_act_model_layers_0_mlp/mean': '0.209', 'train/tensor_act_model_layers_0_mlp/std': '0.332', 'train/tensor_act_model_layers_0_mlp/max_abs': '1.594', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.5664', 'train/tensor_act_model_layers_0_mlp/max': '1.594', 'train/tensor_act_model_layers_0_mlp/range': '2.16', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '836.4', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.209', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.3516', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '1.742', 'train/tensor_act_
+{'loss': '8.12', 'grad_norm': '1.121e+10', 'learning_rate': '9e-05', 'epoch': '0.1982', 'train/total_time_seconds': '134.6', 'train/time_per_step_avg': '0.3108', 'train/epoch_time_elapsed': '159.7', 'train/estimated_remaining_minutes': '0.4273'}
+{'loss': '8.114', 'grad_norm': '1.181e+10', 'learning_rate': '9e-05', 'epoch': '0.1995', 'train/total_time_seconds': '141.1', 'train/time_per_step_avg': '0.3079', 'train/epoch_time_elapsed': '167.5', 'train/estimated_remaining_minutes': '0.429', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001091', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2637', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5391', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '189.7', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0001545', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3848', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3398', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7246', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1443', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.009521', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.498', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '5.188', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.188', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '9.906', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1252', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.004211', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4316', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.125', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-4.969', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.125', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '10.09', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '1026', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.2754', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.418', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '1.906', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6211', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '1.906', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '2.527', 'train/tensor_act_model_layers_0_mlp/norm': '1026', 'train/tensor_act_model_layers_0_mlp/mean': '0.2754', 'train/tensor_act_model_layers_0_mlp/std': '0.418', 'train/tensor_act_model_layers_0_mlp/max_abs': '1.906', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6211', 'train/tensor_act_model_layers_0_mlp/max': '1.906', 'train/tensor_act_model_layers_0_mlp/range': '2.527', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '1053', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.2754', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.4336', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '2.047', 'train/tensor_
+{'loss': '8.108', 'grad_norm': '1.235e+10', 'learning_rate': '9e-05', 'epoch': '0.2009', 'train/total_time_seconds': '147', 'train/time_per_step_avg': '0.3071', 'train/epoch_time_elapsed': '174.4', 'train/estimated_remaining_minutes': '0.4274'}
+{'loss': '8.113', 'grad_norm': '1.288e+10', 'learning_rate': '9e-05', 'epoch': '0.2022', 'train/total_time_seconds': '152.8', 'train/time_per_step_avg': '0.3074', 'train/epoch_time_elapsed': '181.3', 'train/estimated_remaining_minutes': '0.4245'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 16.62it/s]
+ 86%|████████▌ | 3007/3500 [03:04<03:11,  2.57it/s], ?it/s]

zain/Activation/wandb/run-20260814_210109-qauftiuh/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260814_210109-qauftiuh/files/wandb-metadata.json

@@ -0,0 +1,99 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-14T21:01:09.712039Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100l.yaml",
+    "--variants",
+    "glu-relu",
+    "glu-gelu",
+    "glu-sigmoid",
+    "glu-waleed10",
+    "glu-silu-waleed10"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "3fb47c6943d6927eafc39bd399a3d3a520bae74a"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "698779238400"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "nftssf695ll7ar1j38qz3cf5eyic6cod"
+}

zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug-core.log

@@ -0,0 +1,48 @@
+{"time":"2026-08-14T19:38:51.636058907Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp54729boe/port-1375539.txt","pid":1375539,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-14T19:38:51.639970993Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1375539}
+{"time":"2026-08-14T19:38:51.639940209Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1375539-170701-251437063/socket","Net":"unix"}}
+{"time":"2026-08-14T19:38:51.812900736Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-14T19:39:07.276688434Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"2(@)"}
+{"time":"2026-08-14T19:39:07.370077387Z","level":"INFO","msg":"handleInformInit: received","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:39:07.640206016Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:39:13.074249733Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"zj3lg1k64oen"}
+{"time":"2026-08-14T19:51:49.732364673Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"zj3lg1k64oen"}
+{"time":"2026-08-14T19:51:50.223490459Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"zj3lg1k64oen"}
+{"time":"2026-08-14T19:51:50.225275352Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:51:50.225791347Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:51:51.140595835Z","level":"INFO","msg":"handleInformInit: received","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T19:51:51.407779103Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T19:51:56.793135206Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"m2u1nqrk4r4j"}
+{"time":"2026-08-14T20:05:01.389026301Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"m2u1nqrk4r4j"}
+{"time":"2026-08-14T20:05:01.891785946Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"m2u1nqrk4r4j"}
+{"time":"2026-08-14T20:05:01.893356932Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T20:05:01.894067235Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T20:05:03.038763184Z","level":"INFO","msg":"handleInformInit: received","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:05:03.299131712Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:05:08.644597551Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"u5vxhmn1tauv"}
+{"time":"2026-08-14T20:21:26.598738734Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"u5vxhmn1tauv"}
+{"time":"2026-08-14T20:21:27.243676424Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"u5vxhmn1tauv"}
+{"time":"2026-08-14T20:21:27.245262581Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:21:27.245859065Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:21:28.155191217Z","level":"INFO","msg":"handleInformInit: received","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:21:28.414618766Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:21:33.76572887Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"smlkpmpk76rp"}
+{"time":"2026-08-14T20:38:01.401128169Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"smlkpmpk76rp"}
+{"time":"2026-08-14T20:38:02.166866415Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"smlkpmpk76rp"}
+{"time":"2026-08-14T20:38:02.168373578Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:38:02.169204491Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:38:03.474631884Z","level":"INFO","msg":"handleInformInit: received","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:38:03.739404113Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:38:09.1668666Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"9gr96sam7ydd"}
+{"time":"2026-08-14T20:54:40.077063626Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"9gr96sam7ydd"}
+{"time":"2026-08-14T20:54:40.77979146Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"9gr96sam7ydd"}
+{"time":"2026-08-14T20:54:40.781496498Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:54:40.782279361Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.400855251Z","level":"INFO","msg":"connection: closing","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.40094641Z","level":"INFO","msg":"connection: closed successfully","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.400864953Z","level":"INFO","msg":"processOutgoingData: finished","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.400960827Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"2(@)"}
+{"time":"2026-08-14T21:01:09.497626422Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"3(@)"}
+{"time":"2026-08-14T21:01:09.714752804Z","level":"INFO","msg":"handleInformInit: received","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:01:09.994191533Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:01:15.507216386Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"wd49dhcig506"}

zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug-internal.log

@@ -0,0 +1,43 @@
+{"time":"2026-08-14T21:01:09.7150553Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-14T21:01:09.71524213Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-14T21:01:09.994015525Z","level":"INFO","msg":"stream: created new stream","id":"qauftiuh"}
+{"time":"2026-08-14T21:01:09.994100922Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-14T21:01:09.994184934Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-14T21:01:09.994195204Z","level":"INFO","msg":"writer: started","stream_id":"qauftiuh"}
+{"time":"2026-08-14T21:01:09.994219025Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-14T21:01:10.520180246Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:01:10.645642781Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:25.52075425Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":4,"uploaded_len":2}
+{"time":"2026-08-14T21:01:25.702207416Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:26.44643453Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":1531}
+{"time":"2026-08-14T21:01:26.446475996Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-08-14T21:01:26.446859476Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":1564}
+{"time":"2026-08-14T21:01:26.447133828Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":48}
+{"time":"2026-08-14T21:01:26.458822019Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":5762}
+{"time":"2026-08-14T21:01:26.458933657Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":11}
+{"time":"2026-08-14T21:01:26.460243571Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":6248}
+{"time":"2026-08-14T21:01:26.460266471Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-08-14T21:01:26.470709876Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":9671}
+{"time":"2026-08-14T21:01:26.476556775Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":2068}
+{"time":"2026-08-14T21:01:40.5434685Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":3,"events_offset":1,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-14T21:01:41.933404694Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:01:55.532300005Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":2,"events_offset":3,"events_lines":2,"console_offset":3,"console_lines":1}
+{"time":"2026-08-14T21:01:56.286854239Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:10.536990081Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":2,"events_offset":5,"events_lines":2,"console_offset":5,"console_lines":12}
+{"time":"2026-08-14T21:02:11.488046035Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:25.540467013Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":2,"events_offset":7,"events_lines":2,"console_offset":13,"console_lines":1}
+{"time":"2026-08-14T21:02:26.533831445Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:40.539278111Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":2,"events_offset":9,"events_lines":2,"console_offset":17,"console_lines":9}
+{"time":"2026-08-14T21:02:41.487711052Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:55.542783959Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":2,"events_offset":11,"events_lines":2,"console_offset":23,"console_lines":1}
+{"time":"2026-08-14T21:02:56.454929271Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:10.52924582Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":2,"events_offset":13,"events_lines":2,"console_offset":23,"console_lines":1}
+{"time":"2026-08-14T21:03:11.321241037Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:25.537614388Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":15,"events_lines":2,"console_offset":26,"console_lines":11}
+{"time":"2026-08-14T21:03:26.40265823Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:40.541153879Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":2,"events_offset":17,"events_lines":2,"console_offset":33,"console_lines":1}
+{"time":"2026-08-14T21:03:41.456656398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:55.53803109Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":19,"events_lines":2,"console_offset":37,"console_lines":9}
+{"time":"2026-08-14T21:03:56.442504661Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:04:10.540760129Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":21,"events_lines":2,"console_offset":43,"console_lines":1}
+{"time":"2026-08-14T21:04:11.38104672Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug.log

@@ -0,0 +1,23 @@
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_setup.py:_flush():81] Configure stats pid to 936813
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug.log
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug-internal.log
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_init.py:init():772] calling init triggers
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_init.py:init():820] starting backend
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-14 21:01:09,713 INFO    MainThread:936813 [wandb_init.py:init():835] sending inform_init request
+2026-08-14 21:01:09,994 INFO    MainThread:936813 [wandb_init.py:init():840] backend started and connected
+2026-08-14 21:01:09,997 INFO    MainThread:936813 [wandb_init.py:init():910] updated telemetry
+2026-08-14 21:01:10,005 INFO    MainThread:936813 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-14 21:01:10,388 INFO    MainThread:936813 [wandb_init.py:init():978] starting run threads in backend
+2026-08-14 21:01:10,502 INFO    MainThread:936813 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-14 21:01:10,502 INFO    MainThread:936813 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-14 21:01:10,502 INFO    MainThread:936813 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-14 21:01:10,503 INFO    MainThread:936813 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-14 21:01:10,506 INFO    MainThread:936813 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-14 21:01:10,507 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 100, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'relu', 'waleed_beta': 4, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-relu-100L_trash_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 3500, 'learning_rate': 9e-05, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 50, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-relu-100L-16.9M-20260814-210108', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 2498, 'eval_delay': 0, 'per_device_eval_batch_size': 1024, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 50, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-relu-100L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-14 21:01:10,513 INFO    MainThread:936813 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 16934016 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14f25ef82810>>
+2026-08-14 21:01:10,513 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 16934016 None

zain/Activation/wandb/run-20260814_210109-qauftiuh/run-qauftiuh.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccec22def8410d34d0fc8cb07d9c2d0716053748cf5c83516c62d928c7b9965a
+size 26869760

zain/Activation/wandb/run-20260814_210155-rxpj34pj/files/output.log

@@ -0,0 +1,15 @@
+ 10%|█         | 100/1000 [01:12<10:53,  1.38it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '7.169', 'grad_norm': '0.9766', 'learning_rate': '0.001', 'epoch': '0.01078', 'train/total_time_seconds': '6.877', 'train/time_per_step_avg': '0.3438', 'train/epoch_time_elapsed': '14.82', 'train/estimated_remaining_minutes': '5.616'}
+{'loss': '6.014', 'grad_norm': '0.3496', 'learning_rate': '0.001', 'epoch': '0.02156', 'train/total_time_seconds': '13.5', 'train/time_per_step_avg': '0.3374', 'train/epoch_time_elapsed': '29.03', 'train/estimated_remaining_minutes': '5.398'}
+{'loss': '5.569', 'grad_norm': '1.07', 'learning_rate': '0.001', 'epoch': '0.03235', 'train/total_time_seconds': '20.51', 'train/time_per_step_avg': '0.3418', 'train/epoch_time_elapsed': '43.57', 'train/estimated_remaining_minutes': '5.354'}
+{'loss': '5.037', 'grad_norm': '0.5078', 'learning_rate': '0.001', 'epoch': '0.04313', 'train/total_time_seconds': '27.67', 'train/time_per_step_avg': '0.3459', 'train/epoch_time_elapsed': '58.25', 'train/estimated_remaining_minutes': '5.304'}
+{'loss': '4.575', 'grad_norm': '0.4512', 'learning_rate': '0.001', 'epoch': '0.05391', 'train/total_time_seconds': '34.2', 'train/time_per_step_avg': '0.342', 'train/epoch_time_elapsed': '72.42', 'train/estimated_remaining_minutes': '5.129'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 149.81it/s]
+ 20%|█▉        | 195/1000 [02:19<09:41,  1.38it/s]?, ?it/s]
+{'loss': '4.265', 'grad_norm': '0.3926', 'learning_rate': '0.001', 'epoch': '0.06469', 'train/total_time_seconds': '40.89', 'train/time_per_step_avg': '0.3401', 'train/epoch_time_elapsed': '86.77', 'train/estimated_remaining_minutes': '4.997'}
+{'loss': '4.019', 'grad_norm': '0.8477', 'learning_rate': '0.001', 'epoch': '0.07547', 'train/total_time_seconds': '47.33', 'train/time_per_step_avg': '0.3383', 'train/epoch_time_elapsed': '100.7', 'train/estimated_remaining_minutes': '4.846'}
+{'loss': '3.814', 'grad_norm': '0.8281', 'learning_rate': '0.001', 'epoch': '0.08625', 'train/total_time_seconds': '54.25', 'train/time_per_step_avg': '0.3374', 'train/epoch_time_elapsed': '115.1', 'train/estimated_remaining_minutes': '4.747'}
+{'loss': '3.656', 'grad_norm': '1', 'learning_rate': '0.001', 'epoch': '0.09704', 'train/total_time_seconds': '60.76', 'train/time_per_step_avg': '0.3309', 'train/epoch_time_elapsed': '129.1', 'train/estimated_remaining_minutes': '4.613'}

zain/Activation/wandb/run-20260814_210155-rxpj34pj/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260814_210155-rxpj34pj/files/wandb-metadata.json

@@ -0,0 +1,101 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-14T21:01:55.965087Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline.yaml",
+    "--variants",
+    "glu-waleed10",
+    "glu-silu-waleed10",
+    "mlp-waleed10",
+    "mlp-silu-waleed10",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "3fb47c6943d6927eafc39bd399a3d3a520bae74a"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "698779742208"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "nu3b2lpagmx2zf9pq827j2d09jysatwr"
+}

zain/Activation/wandb/run-20260814_210155-rxpj34pj/logs/debug-core.log

@@ -0,0 +1,42 @@
+{"time":"2026-08-14T20:04:25.06988208Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4_27mh86/port-409188.txt","pid":409188,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-14T20:04:25.070357566Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":409188}
+{"time":"2026-08-14T20:04:25.070338944Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-409188-410559-1836801570/socket","Net":"unix"}}
+{"time":"2026-08-14T20:04:25.24795932Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-14T20:04:25.504127391Z","level":"INFO","msg":"handleInformInit: received","streamId":"npumf6lz","id":"1(@)"}
+{"time":"2026-08-14T20:04:25.765319769Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"npumf6lz","id":"1(@)"}
+{"time":"2026-08-14T20:04:31.109816098Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d0p7d03ygyti"}
+{"time":"2026-08-14T20:16:14.526898083Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d0p7d03ygyti"}
+{"time":"2026-08-14T20:16:15.115854524Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"d0p7d03ygyti"}
+{"time":"2026-08-14T20:16:15.117591712Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"npumf6lz","id":"1(@)"}
+{"time":"2026-08-14T20:16:15.118263697Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"npumf6lz","id":"1(@)"}
+{"time":"2026-08-14T20:16:15.759561877Z","level":"INFO","msg":"handleInformInit: received","streamId":"w8mr4w6k","id":"1(@)"}
+{"time":"2026-08-14T20:16:16.020082435Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"w8mr4w6k","id":"1(@)"}
+{"time":"2026-08-14T20:16:21.531557612Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ewwa8fnip5gl"}
+{"time":"2026-08-14T20:28:05.375497777Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ewwa8fnip5gl"}
+{"time":"2026-08-14T20:28:06.097599346Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ewwa8fnip5gl"}
+{"time":"2026-08-14T20:28:06.099103114Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"w8mr4w6k","id":"1(@)"}
+{"time":"2026-08-14T20:28:06.099811223Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"w8mr4w6k","id":"1(@)"}
+{"time":"2026-08-14T20:28:06.637357786Z","level":"INFO","msg":"handleInformInit: received","streamId":"g2ch84q3","id":"1(@)"}
+{"time":"2026-08-14T20:28:06.894723818Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"g2ch84q3","id":"1(@)"}
+{"time":"2026-08-14T20:28:12.337494383Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3viso7y2z8wp"}
+{"time":"2026-08-14T20:39:43.955226503Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3viso7y2z8wp"}
+{"time":"2026-08-14T20:39:44.505666869Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3viso7y2z8wp"}
+{"time":"2026-08-14T20:39:44.507136651Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"g2ch84q3","id":"1(@)"}
+{"time":"2026-08-14T20:39:44.507897996Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"g2ch84q3","id":"1(@)"}
+{"time":"2026-08-14T20:39:45.020279672Z","level":"INFO","msg":"handleInformInit: received","streamId":"j86czemo","id":"1(@)"}
+{"time":"2026-08-14T20:39:45.27941156Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"j86czemo","id":"1(@)"}
+{"time":"2026-08-14T20:39:50.74200388Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ibw5o8e0zw49"}
+{"time":"2026-08-14T20:51:33.789990271Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ibw5o8e0zw49"}
+{"time":"2026-08-14T20:51:34.383490753Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"ibw5o8e0zw49"}
+{"time":"2026-08-14T20:51:34.385061991Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"j86czemo","id":"1(@)"}
+{"time":"2026-08-14T20:51:34.385923822Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"j86czemo","id":"1(@)"}
+{"time":"2026-08-14T20:51:35.100639214Z","level":"INFO","msg":"handleInformInit: received","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T20:51:35.363188061Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T20:51:40.739065686Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:54.898587298Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.393521917Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"8byxg89obfh9"}
+{"time":"2026-08-14T21:01:55.395053333Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.395670668Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"0vnbmyac","id":"1(@)"}
+{"time":"2026-08-14T21:01:55.967276961Z","level":"INFO","msg":"handleInformInit: received","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:01:56.228683674Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rxpj34pj","id":"1(@)"}
+{"time":"2026-08-14T21:02:01.684005979Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"atuum5cul6gx"}

zain/Activation/wandb/run-20260814_210155-rxpj34pj/logs/debug-internal.log

@@ -0,0 +1,27 @@
+{"time":"2026-08-14T21:01:55.967436258Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-14T21:01:55.967577973Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-14T21:01:56.228468944Z","level":"INFO","msg":"stream: created new stream","id":"rxpj34pj"}
+{"time":"2026-08-14T21:01:56.228571423Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-14T21:01:56.228674212Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-14T21:01:56.228689116Z","level":"INFO","msg":"writer: started","stream_id":"rxpj34pj"}
+{"time":"2026-08-14T21:01:56.228712767Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-14T21:01:56.612709801Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:01:56.724524535Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:11.613736811Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-14T21:02:11.755716385Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:26.613440627Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":1,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:02:26.756459811Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:41.612944581Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":3,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:02:41.737144385Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:02:56.61357413Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":5,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:02:56.736287618Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:11.613376401Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":7,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:03:11.772005165Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:26.613829663Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":9,"events_lines":2,"console_offset":2,"console_lines":10}
+{"time":"2026-08-14T21:03:26.755273016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:41.613471478Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":11,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-08-14T21:03:41.725523747Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:03:56.612890048Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":13,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-08-14T21:03:56.723039149Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:04:11.613835461Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":15,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-08-14T21:04:11.764020425Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260814_210155-rxpj34pj/logs/debug.log

@@ -0,0 +1,19 @@
+2026-08-14 21:01:55,966 INFO    MainThread:409188 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_210155-rxpj34pj/logs/debug.log
+2026-08-14 21:01:55,966 INFO    MainThread:409188 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_210155-rxpj34pj/logs/debug-internal.log
+2026-08-14 21:01:55,966 INFO    MainThread:409188 [wandb_init.py:init():772] calling init triggers
+2026-08-14 21:01:55,966 INFO    MainThread:409188 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-14 21:01:55,966 INFO    MainThread:409188 [wandb_init.py:init():820] starting backend
+2026-08-14 21:01:55,966 INFO    MainThread:409188 [wandb_init.py:init():835] sending inform_init request
+2026-08-14 21:01:56,229 INFO    MainThread:409188 [wandb_init.py:init():840] backend started and connected
+2026-08-14 21:01:56,230 INFO    MainThread:409188 [wandb_init.py:init():910] updated telemetry
+2026-08-14 21:01:56,237 INFO    MainThread:409188 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-14 21:01:56,533 INFO    MainThread:409188 [wandb_init.py:init():978] starting run threads in backend
+2026-08-14 21:01:56,607 INFO    MainThread:409188 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-14 21:01:56,608 INFO    MainThread:409188 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-14 21:01:56,608 INFO    MainThread:409188 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-14 21:01:56,608 INFO    MainThread:409188 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-14 21:01:56,609 INFO    MainThread:409188 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-14 21:01:56,609 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 9, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'situglu_low', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-situglu_low-9L_run', 'per_device_train_batch_size': 512, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-situglu_low-9L-2.0M-20260814-210155', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 60000, 'eval_delay': 0, 'per_device_eval_batch_size': 512, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/A-glu-situglu_low-9L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-14 21:01:56,611 INFO    MainThread:409188 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 2001280 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x15413e1378d0>>
+2026-08-14 21:01:56,611 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 2001280 None