Auto upload zain 2026-08-19T17:15:02.582332 (part 3)

zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/config.yaml

@@ -0,0 +1,435 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            rjsr5aiivqlwnjxtkxlgjkrgdfxgvs5f:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100L.yaml
+                    - --variants
+                    - mlp-linear-3L
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "708477448192"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 463c9961366755fc55f02df9a0d471b3cbcb025e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-19T17:12:15.316941Z"
+                writerId: rjsr5aiivqlwnjxtkxlgjkrgdfxgvs5f
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: linear
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 100
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-mlp-linear-3L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.001
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: mlp
+model/num_parameters:
+    value: 1016704
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 3
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/mlp-linear-3L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 1500
+per_device_train_batch_size:
+    value: 80
+powlu_m:
+    value: 3
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-mlp-linear-3L-1.0M-20260819-171214
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 100
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/output.log

@@ -88,8 +88,34 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 340
   - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
   - If you are not the owner of the model architecture class, please contact the model code owner to update it.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 338.44it/s]
- 89%|████████▉ | 889/1000 [02:09<00:08, 13.32it/s]?, ?it/s]
+ 90%|█████████ | 900/1000 [02:18<00:07, 13.35[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '2.927', 'grad_norm': '0.7422', 'learning_rate': '0.001', 'epoch': '0.06909', 'train/total_time_seconds': '14.41', 'train/time_per_step_avg': '0.01651', 'train/epoch_time_elapsed': '124.6', 'train/estimated_remaining_minutes': '0.05273'}
 {'loss': '2.923', 'grad_norm': '0.8828', 'learning_rate': '0.001', 'epoch': '0.07078', 'train/total_time_seconds': '14.74', 'train/time_per_step_avg': '0.01652', 'train/epoch_time_elapsed': '126.2', 'train/estimated_remaining_minutes': '0.04681'}
 {'loss': '2.909', 'grad_norm': '0.8008', 'learning_rate': '0.001', 'epoch': '0.07246', 'train/total_time_seconds': '15.08', 'train/time_per_step_avg': '0.01655', 'train/epoch_time_elapsed': '127.7', 'train/estimated_remaining_minutes': '0.04091'}
 {'loss': '2.907', 'grad_norm': '1.133', 'learning_rate': '0.001', 'epoch': '0.07415', 'train/total_time_seconds': '15.41', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '129.2', 'train/estimated_remaining_minutes': '0.03502'}
+{'loss': '2.868', 'grad_norm': '0.9922', 'learning_rate': '0.001', 'epoch': '0.07583', 'train/total_time_seconds': '15.74', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '130.7', 'train/estimated_remaining_minutes': '0.02914'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.89', 'eval_runtime': '7.735', 'eval_samples_per_second': '1232', 'eval_steps_per_second': '0.905', 'epoch': '0.07583', 'train/total_time_seconds': '15.74', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '138.4', 'train/estimated_remaining_minutes': '0.02914'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 326.63it/s]
+100%|██████████| 1000/1000 [02:33<00:00, 11.6[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.875', 'grad_norm': '0.8125', 'learning_rate': '0.001', 'epoch': '0.07752', 'train/total_time_seconds': '16.07', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '140', 'train/estimated_remaining_minutes': '0.02329'}
+{'loss': '2.869', 'grad_norm': '0.8047', 'learning_rate': '0.001', 'epoch': '0.0792', 'train/total_time_seconds': '16.4', 'train/time_per_step_avg': '0.01656', 'train/epoch_time_elapsed': '141.5', 'train/estimated_remaining_minutes': '0.01745'}
+{'loss': '2.866', 'grad_norm': '0.8867', 'learning_rate': '0.001', 'epoch': '0.08089', 'train/total_time_seconds': '16.73', 'train/time_per_step_avg': '0.01653', 'train/epoch_time_elapsed': '143', 'train/estimated_remaining_minutes': '0.01162'}
+{'loss': '2.861', 'grad_norm': '0.7383', 'learning_rate': '0.001', 'epoch': '0.08257', 'train/total_time_seconds': '17.06', 'train/time_per_step_avg': '0.01653', 'train/epoch_time_elapsed': '144.5', 'train/estimated_remaining_minutes': '0.005803'}
+{'loss': '2.839', 'grad_norm': '0.8867', 'learning_rate': '0.001', 'epoch': '0.08426', 'train/total_time_seconds': '17.39', 'train/time_per_step_avg': '0.01654', 'train/epoch_time_elapsed': '146', 'train/estimated_remaining_minutes': '0'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '2.841', 'eval_runtime': '7.595', 'eval_samples_per_second': '1254', 'eval_steps_per_second': '0.922', 'epoch': '0.08426', 'train/total_time_seconds': '17.39', 'train/time_per_step_avg': '0.01654', 'train/epoch_time_elapsed': '153.6', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 184.90it/s]
+100%|██████████| 1000/1000 [02:33<00:00,  6.51it/s], ?it/s]
+{'train_runtime': '154.6', 'train_samples_per_second': '517.5', 'train_steps_per_second': '6.468', 'train_loss': '3.728', 'epoch': '0.08426', 'train/total_time_seconds': '17.39', 'train/time_per_step_avg': '0.01654', 'train/epoch_time_elapsed': '153.7', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 7/7 [00:05<00:00,  1.32it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 324.94it/s]
+>>> FINISHED mlp-linear-3L successfully

zain/Activation/wandb/run-20260819_171215-2nil4rqn/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train_steps_per_second":6.468,"_wandb":{"runtime":161},"train/learning_rate":0.001,"train/global_step":1000,"eval/steps_per_second":0.927,"train/epoch":0.08426019548365352,"_timestamp":1.787159697355621e+09,"train_runtime":154.6026,"_runtime":161,"train/loss":2.8389766693115233,"train/grad_norm":0.88671875,"eval/loss":2.841240167617798,"eval/runtime":7.5505,"train_loss":3.7283405075073244,"eval/samples_per_second":1261.772,"_step":61,"total_flos":1.2101615616e+14,"train_samples_per_second":517.456}

zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-core.log

@@ -54,3 +54,11 @@
 {"time":"2026-08-19T17:12:15.319455303Z","level":"INFO","msg":"handleInformInit: received","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:15.582732337Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2nil4rqn","id":"6(@)"}
 {"time":"2026-08-19T17:12:21.153010638Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:57.367689028Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.279859891Z","level":"INFO","msg":"connection: cancelling request","id":"6(@)","requestId":"g8ym1re70rep"}
+{"time":"2026-08-19T17:14:58.28171116Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:14:58.282403203Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2nil4rqn","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260323511Z","level":"INFO","msg":"connection: closing","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.26039463Z","level":"INFO","msg":"connection: closed successfully","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260332968Z","level":"INFO","msg":"processOutgoingData: finished","id":"6(@)"}
+{"time":"2026-08-19T17:15:00.260403826Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"6(@)"}

zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug-internal.log

@@ -23,3 +23,15 @@
 {"time":"2026-08-19T17:14:01.314421382Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-19T17:14:16.125013671Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":6,"events_offset":13,"events_lines":2,"console_offset":68,"console_lines":1}
 {"time":"2026-08-19T17:14:16.73098493Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:31.124676234Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":6,"events_offset":15,"events_lines":2,"console_offset":74,"console_lines":23}
+{"time":"2026-08-19T17:14:31.25448083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:46.124973259Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":6,"events_offset":17,"events_lines":2,"console_offset":90,"console_lines":1}
+{"time":"2026-08-19T17:14:46.269794228Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:58.136247949Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-19T17:14:58.136517121Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":3,"events_offset":19,"events_lines":1,"console_offset":96,"console_lines":25,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-19T17:14:58.277374581Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-19T17:14:58.278584446Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-19T17:14:58.281745009Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-19T17:14:58.281775882Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-19T17:14:58.281829879Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-19T17:14:58.281833448Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260819_171215-2nil4rqn/logs/debug.log

@@ -21,3 +21,8 @@ config: {'_wandb': {}}
 2026-08-19 17:12:16,120 INFO    MainThread:3920475 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 3, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'mlp', 'activation': 'linear', 'waleed_beta': 10.0, 'powlu_m': 3.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/mlp-linear-3L_run', 'per_device_train_batch_size': 80, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-mlp-linear-3L-1.0M-20260819-171214', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 100, 'eval_delay': 0, 'per_device_eval_batch_size': 1500, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-mlp-linear-3L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-19 17:12:16,122 INFO    MainThread:3920475 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 1016704 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14f219615650>>
 2026-08-19 17:12:16,122 INFO    MainThread:3920475 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 1016704 None
+2026-08-19 17:14:57,366 INFO    MainThread:3920475 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research-ultimate-checking/2nil4rqn
+2026-08-19 17:14:57,367 INFO    MainThread:3920475 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-19 17:14:57,367 INFO    MainThread:3920475 [wandb_run.py:_restore():2570] restore
+2026-08-19 17:14:57,367 INFO    MainThread:3920475 [wandb_run.py:_restore():2576] restore done
+2026-08-19 17:14:58,281 INFO    MainThread:3920475 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260819_171215-2nil4rqn/run-2nil4rqn.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eabb4a030bca62aea6f627cbe7cd12ff2a9480104eafd775ab756749b9220aa4
-size 196608
+oid sha256:a7b5da609b9fccdc3e7dd6fde9d7d49ec5a9b66eb9fe3dc139ddf206bfab728d
+size 236528