Auto upload zain 2026-08-14T21:19:40.486080 (part 4)

.gitattributes

@@ -57,3 +57,5 @@ zain/Activation/wandb/run-20260814_210155-rxpj34pj/run-rxpj34pj.wandb filter=lfs
 zain/Activation/wandb/run-20260814_210809-a6574zx8/run-a6574zx8.wandb filter=lfs diff=lfs merge=lfs -text
 zain/Activation/out/glu-gelu-100L_trash_run/training_log.jsonl filter=lfs diff=lfs merge=lfs -text
 zain/Activation/out/glu-sigmoid-100L_trash_run/training_log.jsonl filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260814_211407-i85cngas/run-i85cngas.wandb filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260814_211507-54wnyexz/run-54wnyexz.wandb filter=lfs diff=lfs merge=lfs -text

zain/Activation/wandb/debug-internal.log

@@ -1,12 +1,65 @@
-{"time":"2026-08-14T21:14:07.911227197Z","level":"INFO","msg":"wandb-core"}
-{"time":"2026-08-14T21:14:07.911377478Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
-{"time":"2026-08-14T21:14:08.168044483Z","level":"INFO","msg":"stream: created new stream","id":"i85cngas"}
-{"time":"2026-08-14T21:14:08.16812752Z","level":"INFO","msg":"handler: started"}
-{"time":"2026-08-14T21:14:08.168213295Z","level":"INFO","msg":"stream: started"}
-{"time":"2026-08-14T21:14:08.168225298Z","level":"INFO","msg":"writer: started","stream_id":"i85cngas"}
-{"time":"2026-08-14T21:14:08.168254998Z","level":"INFO","msg":"sender: started"}
-{"time":"2026-08-14T21:14:08.675326195Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
-{"time":"2026-08-14T21:14:08.803069742Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:14:23.675538684Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":2,"uploaded_len":2}
-{"time":"2026-08-14T21:14:23.78531736Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:14:38.67602145Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":1,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:15:07.169424866Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-14T21:15:07.169567468Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-14T21:15:07.435532929Z","level":"INFO","msg":"stream: created new stream","id":"54wnyexz"}
+{"time":"2026-08-14T21:15:07.435601691Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-14T21:15:07.435689178Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-14T21:15:07.43571596Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-14T21:15:07.435698857Z","level":"INFO","msg":"writer: started","stream_id":"54wnyexz"}
+{"time":"2026-08-14T21:15:07.865306518Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":2}
+{"time":"2026-08-14T21:15:07.979243259Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:22.8655656Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-14T21:15:23.014629997Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:23.378577725Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":1538}
+{"time":"2026-08-14T21:15:23.378611071Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-08-14T21:15:23.38845317Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":4102}
+{"time":"2026-08-14T21:15:23.388483609Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-08-14T21:15:23.389724048Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":4603}
+{"time":"2026-08-14T21:15:23.389898245Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":36}
+{"time":"2026-08-14T21:15:23.39094459Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":5037}
+{"time":"2026-08-14T21:15:23.391389079Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":103}
+{"time":"2026-08-14T21:15:23.391844857Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":5345}
+{"time":"2026-08-14T21:15:23.398774475Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":2645}
+{"time":"2026-08-14T21:15:23.399288691Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":8229}
+{"time":"2026-08-14T21:15:23.400206063Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":379}
+{"time":"2026-08-14T21:15:23.400644516Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":8832}
+{"time":"2026-08-14T21:15:23.400737823Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":19}
+{"time":"2026-08-14T21:15:23.400946664Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":8941}
+{"time":"2026-08-14T21:15:23.401087444Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":37}
+{"time":"2026-08-14T21:15:23.405809502Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":10867}
+{"time":"2026-08-14T21:15:23.405942913Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":22}
+{"time":"2026-08-14T21:15:23.407538574Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":11526}
+{"time":"2026-08-14T21:15:23.408103184Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":213}
+{"time":"2026-08-14T21:15:37.883634237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":2,"events_offset":1,"events_lines":2,"console_offset":1,"console_lines":1}
+{"time":"2026-08-14T21:15:38.603529933Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:52.884551759Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":3,"events_offset":3,"events_lines":2,"console_offset":1,"console_lines":1}
+{"time":"2026-08-14T21:15:53.827374416Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:07.891440937Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":2,"events_offset":5,"events_lines":2,"console_offset":3,"console_lines":12}
+{"time":"2026-08-14T21:16:08.876461742Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:22.890274165Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":2,"events_offset":7,"events_lines":2,"console_offset":11,"console_lines":1}
+{"time":"2026-08-14T21:16:23.704159399Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:37.88767292Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":2,"events_offset":9,"events_lines":2,"console_offset":15,"console_lines":9}
+{"time":"2026-08-14T21:16:38.690316648Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:52.886427045Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":2,"events_offset":11,"events_lines":2,"console_offset":21,"console_lines":1}
+{"time":"2026-08-14T21:16:54.006407739Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:07.874889179Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":2,"events_offset":13,"events_lines":2,"console_offset":21,"console_lines":1}
+{"time":"2026-08-14T21:17:08.765471891Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:22.884402781Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":15,"events_lines":2,"console_offset":24,"console_lines":11}
+{"time":"2026-08-14T21:17:23.782618431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:37.894674616Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":2,"events_offset":17,"events_lines":2,"console_offset":31,"console_lines":1}
+{"time":"2026-08-14T21:17:38.741283006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:52.887547658Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":19,"events_lines":2,"console_offset":35,"console_lines":9}
+{"time":"2026-08-14T21:17:53.759903107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:07.894710599Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":21,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-08-14T21:18:08.817504637Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:22.875702416Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":2,"events_offset":23,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-08-14T21:18:23.642998715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:37.887794344Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":2,"events_offset":25,"events_lines":2,"console_offset":44,"console_lines":11}
+{"time":"2026-08-14T21:18:38.696615712Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:52.883505832Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":2,"events_offset":27,"events_lines":2,"console_offset":51,"console_lines":1}
+{"time":"2026-08-14T21:18:53.743917259Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:07.888490889Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":2,"events_offset":29,"events_lines":2,"console_offset":55,"console_lines":9}
+{"time":"2026-08-14T21:19:08.707657114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:22.884734385Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":2,"events_offset":31,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-08-14T21:19:23.790343089Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:37.875793912Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":2,"events_offset":33,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-08-14T21:19:38.585616478Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/debug.log

@@ -1,19 +1,20 @@
-2026-08-14 21:14:07,909 INFO    MainThread:409188 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_211407-i85cngas/logs/debug.log
-2026-08-14 21:14:07,910 INFO    MainThread:409188 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_211407-i85cngas/logs/debug-internal.log
-2026-08-14 21:14:07,910 INFO    MainThread:409188 [wandb_init.py:init():772] calling init triggers
-2026-08-14 21:14:07,910 INFO    MainThread:409188 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug.log
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug-internal.log
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():772] calling init triggers
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2026-08-14 21:14:07,910 INFO    MainThread:409188 [wandb_init.py:init():820] starting backend
-2026-08-14 21:14:07,910 INFO    MainThread:409188 [wandb_init.py:init():835] sending inform_init request
-2026-08-14 21:14:08,168 INFO    MainThread:409188 [wandb_init.py:init():840] backend started and connected
-2026-08-14 21:14:08,169 INFO    MainThread:409188 [wandb_init.py:init():910] updated telemetry
-2026-08-14 21:14:08,176 INFO    MainThread:409188 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
-2026-08-14 21:14:08,594 INFO    MainThread:409188 [wandb_init.py:init():978] starting run threads in backend
-2026-08-14 21:14:08,670 INFO    MainThread:409188 [wandb_run.py:_console_start():2621] atexit reg
-2026-08-14 21:14:08,670 INFO    MainThread:409188 [wandb_run.py:_redirect():2471] redirect: wrap_raw
-2026-08-14 21:14:08,670 INFO    MainThread:409188 [wandb_run.py:_redirect():2540] Wrapping output streams.
-2026-08-14 21:14:08,670 INFO    MainThread:409188 [wandb_run.py:_redirect():2563] Redirects installed.
-2026-08-14 21:14:08,671 INFO    MainThread:409188 [wandb_init.py:init():1016] run started, returning control to user process
-2026-08-14 21:14:08,672 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 9, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'waleedglu_low', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-waleedglu_low-9L_run', 'per_device_train_batch_size': 512, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-waleedglu_low-9L-2.0M-20260814-211407', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 60000, 'eval_delay': 0, 'per_device_eval_batch_size': 512, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/A-glu-waleedglu_low-9L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
-2026-08-14 21:14:08,674 INFO    MainThread:409188 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 2001280 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x153f494c3450>>
-2026-08-14 21:14:08,674 INFO    MainThread:409188 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 2001280 None
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():820] starting backend
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():835] sending inform_init request
+2026-08-14 21:15:07,436 INFO    MainThread:936813 [wandb_init.py:init():840] backend started and connected
+2026-08-14 21:15:07,437 INFO    MainThread:936813 [wandb_init.py:init():910] updated telemetry
+2026-08-14 21:15:07,444 INFO    MainThread:936813 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-14 21:15:07,772 INFO    MainThread:936813 [wandb_init.py:init():978] starting run threads in backend
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-14 21:15:07,852 INFO    MainThread:936813 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-14 21:15:07,853 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 100, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'sigmoid', 'waleed_beta': 4, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-sigmoid-100L_trash_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 3500, 'learning_rate': 9e-05, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 50, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-sigmoid-100L-16.9M-20260814-211506', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 2498, 'eval_delay': 0, 'per_device_eval_batch_size': 1024, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 50, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-sigmoid-100L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-14 21:15:07,858 INFO    MainThread:936813 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 16934016 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee39eebe10>>
+2026-08-14 21:15:07,858 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 16934016 None

zain/Activation/wandb/run-20260814_193907-kk1xbqht/logs/debug-core.log

@@ -54,3 +54,10 @@
 {"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
 {"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
 {"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_195151-ix6wcbwv/logs/debug-core.log

@@ -54,3 +54,10 @@
 {"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
 {"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
 {"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_200503-bdhnr27j/logs/debug-core.log

@@ -54,3 +54,10 @@
 {"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
 {"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
 {"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_202128-twgo1e89/logs/debug-core.log

@@ -54,3 +54,10 @@
 {"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
 {"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
 {"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_203803-7p5140bq/logs/debug-core.log

@@ -54,3 +54,10 @@
 {"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
 {"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
 {"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug-core.log

@@ -54,3 +54,10 @@
 {"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
 {"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
 {"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_210809-a6574zx8/files/config.yaml

@@ -0,0 +1,439 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            upcn9ahkwrhy5vr293v2ixhfn8n9p8k0:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100l.yaml
+                    - --variants
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "701339574272"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T21:08:09.034947Z"
+                writerId: upcn9ahkwrhy5vr293v2ixhfn8n9p8k0
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 41
+                - 61
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: gelu
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 2498
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-glu-gelu-100L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 9e-05
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 3500
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 16934016
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 100
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-gelu-100L_trash_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 1024
+per_device_train_batch_size:
+    value: 64
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-gelu-100L-16.9M-20260814-210807
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 50
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 4
+warmup_steps:
+    value: 50
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260814_210809-a6574zx8/files/output.log

@@ -102,3 +102,10 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 16.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.61it/s]
 100%|██████████| 3500/3500 [06:23<00:00,  9.12it/s], ?it/s]
 {'train_runtime': '385', 'train_samples_per_second': '581.9', 'train_steps_per_second': '9.091', 'train_loss': '2.202', 'epoch': '0.2359', 'train/total_time_seconds': '308.9', 'train/time_per_step_avg': '0.2828', 'train/epoch_time_elapsed': '383.8', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 10/10 [00:15<00:00,  1.51s/it]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 16.64it/s]
+>>> FINISHED glu-gelu-100L successfully

zain/Activation/wandb/run-20260814_210809-a6574zx8/logs/debug-core.log

@@ -54,3 +54,10 @@
 {"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
 {"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
 {"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_210809-a6574zx8/logs/debug-internal.log

@@ -59,3 +59,13 @@
 {"time":"2026-08-14T21:14:10.945223769Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-14T21:14:24.907797301Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":1,"events_offset":47,"events_lines":2,"console_offset":91,"console_lines":1}
 {"time":"2026-08-14T21:14:25.747074137Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:14:39.910134146Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":50,"history_lines":2,"events_offset":49,"events_lines":2,"console_offset":91,"console_lines":1}
+{"time":"2026-08-14T21:14:40.657863657Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:14:52.017579403Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-14T21:14:52.026614872Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":52,"history_lines":1,"events_offset":51,"events_lines":1,"console_offset":96,"console_lines":15,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-14T21:14:52.832575952Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:14:52.833971568Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-14T21:14:52.873689012Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-14T21:14:52.873727527Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-14T21:14:52.873803974Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-14T21:14:52.873808726Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260814_210809-a6574zx8/logs/debug.log

@@ -18,3 +18,8 @@ config: {'_wandb': {}}
 2026-08-14 21:08:09,886 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 100, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'gelu', 'waleed_beta': 4, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-gelu-100L_trash_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 3500, 'learning_rate': 9e-05, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 50, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-gelu-100L-16.9M-20260814-210807', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 2498, 'eval_delay': 0, 'per_device_eval_batch_size': 1024, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 50, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-gelu-100L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
 2026-08-14 21:08:09,891 INFO    MainThread:936813 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 16934016 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee39e0dd90>>
 2026-08-14 21:08:09,891 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 16934016 None
+2026-08-14 21:14:51,273 INFO    MainThread:936813 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research7/a6574zx8
+2026-08-14 21:14:51,274 INFO    MainThread:936813 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-14 21:14:51,274 INFO    MainThread:936813 [wandb_run.py:_restore():2570] restore
+2026-08-14 21:14:51,274 INFO    MainThread:936813 [wandb_run.py:_restore():2576] restore done
+2026-08-14 21:14:52,872 INFO    MainThread:936813 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260814_210809-a6574zx8/run-a6574zx8.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3fadffc1235472998d6b03b8d907af7ec4e5b94a9fce797b5be85ac572592c60
-size 52264960
+oid sha256:6d299e1b6e5245f9b4c77e34837e862f629cd6e5e8bc9f8e4606f02fcc51a163
+size 52308490

zain/Activation/wandb/run-20260814_211407-i85cngas/files/output.log

@@ -1,3 +1,45 @@
-  4%|▍         | 44/1000 [00:29<09:55,  1.61it/s]
+ 10%|█         | 100/1000 [01:02<09:54,  1.51it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '7.143', 'grad_norm': '0.9727', 'learning_rate': '0.001', 'epoch': '0.01078', 'train/total_time_seconds': '5.76', 'train/time_per_step_avg': '0.288', 'train/epoch_time_elapsed': '13.8', 'train/estimated_remaining_minutes': '4.704'}
 {'loss': '5.904', 'grad_norm': '0.8125', 'learning_rate': '0.001', 'epoch': '0.02156', 'train/total_time_seconds': '11.67', 'train/time_per_step_avg': '0.2917', 'train/epoch_time_elapsed': '27.3', 'train/estimated_remaining_minutes': '4.668'}
+{'loss': '5.334', 'grad_norm': '0.75', 'learning_rate': '0.001', 'epoch': '0.03235', 'train/total_time_seconds': '16.46', 'train/time_per_step_avg': '0.2744', 'train/epoch_time_elapsed': '39.57', 'train/estimated_remaining_minutes': '4.299'}
+{'loss': '4.782', 'grad_norm': '0.4121', 'learning_rate': '0.001', 'epoch': '0.04313', 'train/total_time_seconds': '20.63', 'train/time_per_step_avg': '0.2579', 'train/epoch_time_elapsed': '51.19', 'train/estimated_remaining_minutes': '3.954'}
+{'loss': '4.378', 'grad_norm': '0.6055', 'learning_rate': '0.001', 'epoch': '0.05391', 'train/total_time_seconds': '24.69', 'train/time_per_step_avg': '0.2469', 'train/epoch_time_elapsed': '62.89', 'train/estimated_remaining_minutes': '3.703'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 154.20it/s]
+ 20%|██        | 200/1000 [02:13<09:01,  1.48it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '4.12', 'grad_norm': '1.125', 'learning_rate': '0.001', 'epoch': '0.06469', 'train/total_time_seconds': '31.46', 'train/time_per_step_avg': '0.257', 'train/epoch_time_elapsed': '77.2', 'train/estimated_remaining_minutes': '3.845'}
+{'loss': '3.904', 'grad_norm': '0.8047', 'learning_rate': '0.001', 'epoch': '0.07547', 'train/total_time_seconds': '37.8', 'train/time_per_step_avg': '0.2613', 'train/epoch_time_elapsed': '91.15', 'train/estimated_remaining_minutes': '3.87'}
+{'loss': '3.715', 'grad_norm': '1.258', 'learning_rate': '0.001', 'epoch': '0.08625', 'train/total_time_seconds': '44.08', 'train/time_per_step_avg': '0.2762', 'train/epoch_time_elapsed': '105', 'train/estimated_remaining_minutes': '3.857'}
+{'loss': '3.565', 'grad_norm': '0.6289', 'learning_rate': '0.001', 'epoch': '0.09704', 'train/total_time_seconds': '50.42', 'train/time_per_step_avg': '0.2979', 'train/epoch_time_elapsed': '118.9', 'train/estimated_remaining_minutes': '3.828'}
+{'loss': '3.437', 'grad_norm': '0.5781', 'learning_rate': '0.001', 'epoch': '0.1078', 'train/total_time_seconds': '57.4', 'train/time_per_step_avg': '0.3272', 'train/epoch_time_elapsed': '133.4', 'train/estimated_remaining_minutes': '3.827'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 161.42it/s]
+ 30%|███       | 300/1000 [03:23<07:49,  1.49it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '3.323', 'grad_norm': '0.832', 'learning_rate': '0.001', 'epoch': '0.1186', 'train/total_time_seconds': '64.54', 'train/time_per_step_avg': '0.3309', 'train/epoch_time_elapsed': '148', 'train/estimated_remaining_minutes': '3.814'}
+{'loss': '3.229', 'grad_norm': '0.6211', 'learning_rate': '0.001', 'epoch': '0.1294', 'train/total_time_seconds': '70.94', 'train/time_per_step_avg': '0.3314', 'train/epoch_time_elapsed': '161.9', 'train/estimated_remaining_minutes': '3.744'}
+{'loss': '3.139', 'grad_norm': '0.5859', 'learning_rate': '0.001', 'epoch': '0.1402', 'train/total_time_seconds': '77.43', 'train/time_per_step_avg': '0.3335', 'train/epoch_time_elapsed': '175.9', 'train/estimated_remaining_minutes': '3.673'}
+{'loss': '3.056', 'grad_norm': '0.9492', 'learning_rate': '0.001', 'epoch': '0.1509', 'train/total_time_seconds': '83.82', 'train/time_per_step_avg': '0.3341', 'train/epoch_time_elapsed': '189.9', 'train/estimated_remaining_minutes': '3.592'}
+{'loss': '2.987', 'grad_norm': '0.4531', 'learning_rate': '0.001', 'epoch': '0.1617', 'train/total_time_seconds': '90.28', 'train/time_per_step_avg': '0.3288', 'train/epoch_time_elapsed': '203.7', 'train/estimated_remaining_minutes': '3.511'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 156.30it/s]
+ 40%|████      | 400/1000 [04:34<07:23,  1.35it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.924', 'grad_norm': '0.5938', 'learning_rate': '0.001', 'epoch': '0.1725', 'train/total_time_seconds': '96.87', 'train/time_per_step_avg': '0.3232', 'train/epoch_time_elapsed': '217.9', 'train/estimated_remaining_minutes': '3.431'}
+{'loss': '2.863', 'grad_norm': '0.6836', 'learning_rate': '0.001', 'epoch': '0.1833', 'train/total_time_seconds': '103.1', 'train/time_per_step_avg': '0.3219', 'train/epoch_time_elapsed': '231.7', 'train/estimated_remaining_minutes': '3.336'}
+{'loss': '2.815', 'grad_norm': '0.582', 'learning_rate': '0.001', 'epoch': '0.1941', 'train/total_time_seconds': '109.8', 'train/time_per_step_avg': '0.3233', 'train/epoch_time_elapsed': '245.9', 'train/estimated_remaining_minutes': '3.252'}
+{'loss': '2.769', 'grad_norm': '0.6719', 'learning_rate': '0.001', 'epoch': '0.2049', 'train/total_time_seconds': '116.7', 'train/time_per_step_avg': '0.3286', 'train/epoch_time_elapsed': '260.4', 'train/estimated_remaining_minutes': '3.173'}
+{'loss': '2.726', 'grad_norm': '0.5781', 'learning_rate': '0.001', 'epoch': '0.2156', 'train/total_time_seconds': '123.6', 'train/time_per_step_avg': '0.3337', 'train/epoch_time_elapsed': '274.8', 'train/estimated_remaining_minutes': '3.091'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 164.59it/s]
+ 49%|████▉     | 493/1000 [05:39<06:06,  1.38it/s]?, ?it/s]
+{'loss': '2.691', 'grad_norm': '0.7578', 'learning_rate': '0.001', 'epoch': '0.2264', 'train/total_time_seconds': '130.1', 'train/time_per_step_avg': '0.3322', 'train/epoch_time_elapsed': '288.8', 'train/estimated_remaining_minutes': '2.994'}
+{'loss': '2.668', 'grad_norm': '0.6133', 'learning_rate': '0.001', 'epoch': '0.2372', 'train/total_time_seconds': '136.4', 'train/time_per_step_avg': '0.333', 'train/epoch_time_elapsed': '302.7', 'train/estimated_remaining_minutes': '2.894'}
+{'loss': '2.632', 'grad_norm': '0.7461', 'learning_rate': '0.001', 'epoch': '0.248', 'train/total_time_seconds': '142.8', 'train/time_per_step_avg': '0.3306', 'train/epoch_time_elapsed': '316.7', 'train/estimated_remaining_minutes': '2.794'}
+{'loss': '2.6', 'grad_norm': '0.6758', 'learning_rate': '0.001', 'epoch': '0.2588', 'train/total_time_seconds': '149.2', 'train/time_per_step_avg': '0.3248', 'train/epoch_time_elapsed': '330.5', 'train/estimated_remaining_minutes': '2.693'}

zain/Activation/wandb/run-20260814_211407-i85cngas/logs/debug-internal.log

@@ -11,3 +11,43 @@
 {"time":"2026-08-14T21:14:23.78531736Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
 {"time":"2026-08-14T21:14:38.67602145Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":1,"events_lines":2,"console_offset":0,"console_lines":1}
 {"time":"2026-08-14T21:14:39.034739883Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:14:53.675822723Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":3,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:14:53.807395658Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:08.675941416Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":5,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:15:08.796615184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:23.67592928Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":7,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:15:23.823808371Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:38.676060528Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":9,"events_lines":2,"console_offset":2,"console_lines":10}
+{"time":"2026-08-14T21:15:38.800433515Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:53.676379163Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":11,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-08-14T21:15:53.864573795Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:08.676451313Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":13,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-08-14T21:16:08.846470743Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:23.676410533Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":15,"events_lines":2,"console_offset":10,"console_lines":1}
+{"time":"2026-08-14T21:16:23.824402577Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:38.676105689Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":17,"events_lines":2,"console_offset":12,"console_lines":10}
+{"time":"2026-08-14T21:16:38.797045956Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:53.675927055Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":19,"events_lines":2,"console_offset":20,"console_lines":1}
+{"time":"2026-08-14T21:16:53.988742485Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:08.67565261Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":21,"events_lines":2,"console_offset":20,"console_lines":1}
+{"time":"2026-08-14T21:17:08.864252802Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:23.676114146Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":23,"events_lines":2,"console_offset":20,"console_lines":1}
+{"time":"2026-08-14T21:17:23.795170156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:38.675750893Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":25,"events_lines":2,"console_offset":20,"console_lines":1}
+{"time":"2026-08-14T21:17:38.802369133Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:53.676337561Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":27,"events_lines":2,"console_offset":22,"console_lines":10}
+{"time":"2026-08-14T21:17:53.805970984Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:08.675591884Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":29,"events_lines":2,"console_offset":30,"console_lines":1}
+{"time":"2026-08-14T21:18:08.820307105Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:23.675794412Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":31,"events_lines":2,"console_offset":30,"console_lines":1}
+{"time":"2026-08-14T21:18:23.797813599Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:38.67592157Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":33,"events_lines":2,"console_offset":30,"console_lines":1}
+{"time":"2026-08-14T21:18:38.803567824Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:53.675427349Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":35,"events_lines":2,"console_offset":30,"console_lines":1}
+{"time":"2026-08-14T21:18:53.788763708Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:08.675868235Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":37,"events_lines":2,"console_offset":32,"console_lines":10}
+{"time":"2026-08-14T21:19:08.789887478Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:23.67545652Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":39,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-08-14T21:19:23.817761924Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:38.676157583Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":41,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-08-14T21:19:38.796224235Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260814_211507-54wnyexz/files/output.log

@@ -0,0 +1,74 @@
+[Override] Activated with value = -10.0
+ 74%|███████▍  | 2600/3500 [00:37<00:39, 22.64it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '2.273', 'grad_norm': '2.11e+07', 'learning_rate': '3.42e-05', 'epoch': '0.1699', 'train/total_time_seconds': '7.155', 'train/time_per_step_avg': '0.3578', 'train/epoch_time_elapsed': '8.346', 'train/estimated_remaining_minutes': '0.04638', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.6', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001129', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2656', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5488', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001122', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.459', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.459', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8848', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2591', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1885', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.031', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.625', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '757.3', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.0003414', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.2617', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '1.375', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-1.375', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.367', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '2.742', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '197', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.0005646', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.09619', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.6289', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6289', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.5898', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.219', 'train/tensor_act_model_layers_0_mlp/norm': '197', 'train/tensor_act_model_layers_0_mlp/mean': '0.0005646', 'train/tensor_act_model_layers_0_mlp/std': '0.09619', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.6289', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6289', 'train/tensor_act_model_layers_0_mlp/max': '0.5898', 'train/tensor_act_model_layers_0_mlp/range': '1.219', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '287.1', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.001694', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1406', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7383',
+{'loss': '2.34', 'grad_norm': '3.67e+07', 'learning_rate': '7.02e-05', 'epoch': '0.1712', 'train/total_time_seconds': '13.29', 'train/time_per_step_avg': '0.3323', 'train/epoch_time_elapsed': '15.5', 'train/estimated_remaining_minutes': '0.08373'}
+{'loss': '5.932', 'grad_norm': '1.326e+08', 'learning_rate': '9e-05', 'epoch': '0.1726', 'train/total_time_seconds': '19.63', 'train/time_per_step_avg': '0.3272', 'train/epoch_time_elapsed': '22.94', 'train/estimated_remaining_minutes': '0.1202', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.7', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001091', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.4', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001183', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4648', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.4648', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8906', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2584', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1758', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.406', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.094', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.406', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.5', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '780.5', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.0003738', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.2695', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '1.398', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-1.398', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.375', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '2.773', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '203.4', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.01099', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.09863', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.6758', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6758', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.6328', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.309', 'train/tensor_act_model_layers_0_mlp/norm': '203.4', 'train/tensor_act_model_layers_0_mlp/mean': '0.01099', 'train/tensor_act_model_layers_0_mlp/std': '0.09863', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.6758', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6758', 'train/tensor_act_model_layers_0_mlp/max': '0.6328', 'train/tensor_act_model_layers_0_mlp/range': '1.309', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '292.8', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.01215', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1426', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7422', 't
+{'loss': '7.863', 'grad_norm': '2.349e+08', 'learning_rate': '9e-05', 'epoch': '0.1739', 'train/total_time_seconds': '25.74', 'train/time_per_step_avg': '0.3217', 'train/epoch_time_elapsed': '30.09', 'train/estimated_remaining_minutes': '0.153'}
+{'loss': '8.117', 'grad_norm': '3.544e+08', 'learning_rate': '9e-05', 'epoch': '0.1753', 'train/total_time_seconds': '31.87', 'train/time_per_step_avg': '0.3187', 'train/epoch_time_elapsed': '37.23', 'train/estimated_remaining_minutes': '0.1839'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 14.22it/s]
+ 77%|███████▋  | 2700/3500 [01:13<04:29,  2.97it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.233', 'grad_norm': '4.823e+08', 'learning_rate': '9e-05', 'epoch': '0.1766', 'train/total_time_seconds': '38.22', 'train/time_per_step_avg': '0.3106', 'train/epoch_time_elapsed': '44.91', 'train/estimated_remaining_minutes': '0.2139', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.6', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001053', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001129', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3945', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8203', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2607', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1621', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.8867', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.844', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-3.984', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.844', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.828', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '841.2', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.0009842', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.291', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '1.43', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-1.43', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.422', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '2.852', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '229.8', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.02795', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.1089', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.6953', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6953', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.5586', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.254', 'train/tensor_act_model_layers_0_mlp/norm': '229.8', 'train/tensor_act_model_layers_0_mlp/mean': '0.02795', 'train/tensor_act_model_layers_0_mlp/std': '0.1089', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.6953', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6953', 'train/tensor_act_model_layers_0_mlp/max': '0.5586', 'train/tensor_act_model_layers_0_mlp/range': '1.254', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '312.4', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.02905', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1494', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7422', 'tra
+{'loss': '8.297', 'grad_norm': '6.082e+08', 'learning_rate': '9e-05', 'epoch': '0.178', 'train/total_time_seconds': '44.21', 'train/time_per_step_avg': '0.3092', 'train/epoch_time_elapsed': '51.93', 'train/estimated_remaining_minutes': '0.24'}
+{'loss': '8.347', 'grad_norm': '7.256e+08', 'learning_rate': '9e-05', 'epoch': '0.1793', 'train/total_time_seconds': '50.67', 'train/time_per_step_avg': '0.3104', 'train/epoch_time_elapsed': '59.5', 'train/estimated_remaining_minutes': '0.2667', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '188.1', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001144', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.9', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001152', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.4023', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8281', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2621', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1504', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.8906', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.812', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.812', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.812', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '899.6', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.000267', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.3105', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '1.719', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-1.641', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '1.719', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '3.359', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '267.3', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.04395', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.123', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.6836', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6836', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.582', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.266', 'train/tensor_act_model_layers_0_mlp/norm': '267.3', 'train/tensor_act_model_layers_0_mlp/mean': '0.04395', 'train/tensor_act_model_layers_0_mlp/std': '0.123', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.6836', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6836', 'train/tensor_act_model_layers_0_mlp/max': '0.582', 'train/tensor_act_model_layers_0_mlp/range': '1.266', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '341.9', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.04492', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1611', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7539', 'train/tens
+{'loss': '8.382', 'grad_norm': '8.347e+08', 'learning_rate': '9e-05', 'epoch': '0.1807', 'train/total_time_seconds': '56.64', 'train/time_per_step_avg': '0.309', 'train/epoch_time_elapsed': '66.53', 'train/estimated_remaining_minutes': '0.2888'}
+{'loss': '8.44', 'grad_norm': '9.311e+08', 'learning_rate': '9e-05', 'epoch': '0.182', 'train/total_time_seconds': '62.96', 'train/time_per_step_avg': '0.3109', 'train/epoch_time_elapsed': '73.86', 'train/estimated_remaining_minutes': '0.3109'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 17.04it/s]
+ 80%|████████  | 2800/3500 [01:50<03:46,  3.09it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.472', 'grad_norm': '1.04e+09', 'learning_rate': '9e-05', 'epoch': '0.1834', 'train/total_time_seconds': '70.05', 'train/time_per_step_avg': '0.3183', 'train/epoch_time_elapsed': '82.22', 'train/estimated_remaining_minutes': '0.3348', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001205', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001236', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.457', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.457', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8828', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2647', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1396', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.9023', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.5', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-3.984', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.5', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.484', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '969.8', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.0004234', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.334', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '2.281', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-2.25', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '2.281', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '4.531', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '332.1', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.06543', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.1484', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.6953', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.6953', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.5938', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.289', 'train/tensor_act_model_layers_0_mlp/norm': '332.1', 'train/tensor_act_model_layers_0_mlp/mean': '0.06543', 'train/tensor_act_model_layers_0_mlp/std': '0.1484', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.6953', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.6953', 'train/tensor_act_model_layers_0_mlp/max': '0.5938', 'train/tensor_act_model_layers_0_mlp/range': '1.289', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '395.3', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.06689', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1807', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.7891', 'train/tens
+{'loss': '8.497', 'grad_norm': '1.132e+09', 'learning_rate': '9e-05', 'epoch': '0.1847', 'train/total_time_seconds': '76.2', 'train/time_per_step_avg': '0.3199', 'train/epoch_time_elapsed': '89.42', 'train/estimated_remaining_minutes': '0.3523'}
+{'loss': '8.525', 'grad_norm': '1.208e+09', 'learning_rate': '9e-05', 'epoch': '0.186', 'train/total_time_seconds': '82.59', 'train/time_per_step_avg': '0.3191', 'train/epoch_time_elapsed': '96.95', 'train/estimated_remaining_minutes': '0.369', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '188', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001038', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001137', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.4219', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8477', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2677', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1279', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.9141', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.031', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.469', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1056', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.0005379', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.3652', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '2.922', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-2.734', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '2.922', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '5.656', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '443.7', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.1006', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.1924', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '0.8203', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.7188', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '0.8203', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.539', 'train/tensor_act_model_layers_0_mlp/norm': '443.7', 'train/tensor_act_model_layers_0_mlp/mean': '0.1006', 'train/tensor_act_model_layers_0_mlp/std': '0.1924', 'train/tensor_act_model_layers_0_mlp/max_abs': '0.8203', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.7188', 'train/tensor_act_model_layers_0_mlp/max': '0.8203', 'train/tensor_act_model_layers_0_mlp/range': '1.539', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '493.9', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.1016', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.2188', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '0.9766', 'train/ten
+{'loss': '8.553', 'grad_norm': '1.267e+09', 'learning_rate': '9e-05', 'epoch': '0.1874', 'train/total_time_seconds': '88.72', 'train/time_per_step_avg': '0.3209', 'train/epoch_time_elapsed': '104.2', 'train/estimated_remaining_minutes': '0.383'}
+{'loss': '8.564', 'grad_norm': '1.351e+09', 'learning_rate': '9e-05', 'epoch': '0.1887', 'train/total_time_seconds': '94.48', 'train/time_per_step_avg': '0.3152', 'train/epoch_time_elapsed': '110.9', 'train/estimated_remaining_minutes': '0.3936'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 17.10it/s]
+ 83%|████████▎ | 2900/3500 [02:27<03:28,  2.88it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.574', 'grad_norm': '1.443e+09', 'learning_rate': '9e-05', 'epoch': '0.1901', 'train/total_time_seconds': '101', 'train/time_per_step_avg': '0.3093', 'train/epoch_time_elapsed': '118.9', 'train/estimated_remaining_minutes': '0.4058', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.8', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001137', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5547', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.7', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001129', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3867', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8125', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2715', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.126', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.9297', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.25', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '9.125', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1166', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.00174', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4023', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '3.578', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-3.438', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '3.578', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '7.016', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '635.9', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.1602', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.2656', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '1.188', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.7422', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '1.188', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '1.93', 'train/tensor_act_model_layers_0_mlp/norm': '635.9', 'train/tensor_act_model_layers_0_mlp/mean': '0.1602', 'train/tensor_act_model_layers_0_mlp/std': '0.2656', 'train/tensor_act_model_layers_0_mlp/max_abs': '1.188', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.7422', 'train/tensor_act_model_layers_0_mlp/max': '1.188', 'train/tensor_act_model_layers_0_mlp/range': '1.93', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '673.6', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.1611', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.2871', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '1.305', 'train/tensor_act_m
+{'loss': '8.589', 'grad_norm': '1.485e+09', 'learning_rate': '9e-05', 'epoch': '0.1914', 'train/total_time_seconds': '107.2', 'train/time_per_step_avg': '0.3096', 'train/epoch_time_elapsed': '126.2', 'train/estimated_remaining_minutes': '0.4151'}
+{'loss': '8.59', 'grad_norm': '1.544e+09', 'learning_rate': '9e-05', 'epoch': '0.1928', 'train/total_time_seconds': '114.1', 'train/time_per_step_avg': '0.3148', 'train/epoch_time_elapsed': '134.2', 'train/estimated_remaining_minutes': '0.4254', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '188.2', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001305', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2656', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5488', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '191.1', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001266', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4004', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4004', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3887', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.7891', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2748', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1172', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.9414', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.844', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.156', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.844', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '9', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1282', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.003021', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4434', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '4.156', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-3.859', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '4.156', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '8.016', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '891.8', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.2363', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.3652', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '1.641', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.7852', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '1.641', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '2.426', 'train/tensor_act_model_layers_0_mlp/norm': '891.8', 'train/tensor_act_model_layers_0_mlp/mean': '0.2363', 'train/tensor_act_model_layers_0_mlp/std': '0.3652', 'train/tensor_act_model_layers_0_mlp/max_abs': '1.641', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.7852', 'train/tensor_act_model_layers_0_mlp/max': '1.641', 'train/tensor_act_model_layers_0_mlp/range': '2.426', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '920.8', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.2373', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.3809', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '1.695', 'train/tensor_act
+{'loss': '8.597', 'grad_norm': '1.602e+09', 'learning_rate': '9e-05', 'epoch': '0.1941', 'train/total_time_seconds': '119.9', 'train/time_per_step_avg': '0.3118', 'train/epoch_time_elapsed': '141.1', 'train/estimated_remaining_minutes': '0.4302'}
+{'loss': '8.607', 'grad_norm': '1.644e+09', 'learning_rate': '9e-05', 'epoch': '0.1955', 'train/total_time_seconds': '125.7', 'train/time_per_step_avg': '0.3125', 'train/epoch_time_elapsed': '147.9', 'train/estimated_remaining_minutes': '0.4335'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 16.79it/s]
+ 86%|████████▌ | 3000/3500 [03:04<02:46,  3.01it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.605', 'grad_norm': '1.703e+09', 'learning_rate': '9e-05', 'epoch': '0.1968', 'train/total_time_seconds': '132.5', 'train/time_per_step_avg': '0.3149', 'train/epoch_time_elapsed': '155.9', 'train/estimated_remaining_minutes': '0.4386', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001228', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.7', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001419', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.418', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8438', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2795', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.1035', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.9609', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.688', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '9.406', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1418', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.002075', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.4902', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '4.719', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-4.344', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '4.719', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '9.062', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '1195', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.3242', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.4844', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '2.094', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.7695', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '2.094', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '2.863', 'train/tensor_act_model_layers_0_mlp/norm': '1195', 'train/tensor_act_model_layers_0_mlp/mean': '0.3242', 'train/tensor_act_model_layers_0_mlp/std': '0.4844', 'train/tensor_act_model_layers_0_mlp/max_abs': '2.094', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.7695', 'train/tensor_act_model_layers_0_mlp/max': '2.094', 'train/tensor_act_model_layers_0_mlp/range': '2.863', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '1219', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.3262', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.498', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '2.141', 'train/tensor_act
+{'loss': '8.618', 'grad_norm': '1.736e+09', 'learning_rate': '9e-05', 'epoch': '0.1982', 'train/total_time_seconds': '138.1', 'train/time_per_step_avg': '0.3098', 'train/epoch_time_elapsed': '162.6', 'train/estimated_remaining_minutes': '0.4386'}
+{'loss': '8.625', 'grad_norm': '1.778e+09', 'learning_rate': '9e-05', 'epoch': '0.1995', 'train/total_time_seconds': '144.8', 'train/time_per_step_avg': '0.3076', 'train/epoch_time_elapsed': '170.4', 'train/estimated_remaining_minutes': '0.4403', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.8', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.00116', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001213', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3926', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8184', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2880', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.09277', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.9883', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '5.25', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.25', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.844', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '10.09', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1594', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.001572', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.5508', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.156', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.031', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.156', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '10.19', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '1654', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.4648', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.6602', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '2.703', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.793', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '2.703', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '3.496', 'train/tensor_act_model_layers_0_mlp/norm': '1654', 'train/tensor_act_model_layers_0_mlp/mean': '0.4648', 'train/tensor_act_model_layers_0_mlp/std': '0.6602', 'train/tensor_act_model_layers_0_mlp/max_abs': '2.703', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.793', 'train/tensor_act_model_layers_0_mlp/max': '2.703', 'train/tensor_act_model_layers_0_mlp/range': '3.496', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '1673', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.4648', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.6719', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '2.75', 'train/tensor_act_mo
+{'loss': '8.63', 'grad_norm': '1.812e+09', 'learning_rate': '9e-05', 'epoch': '0.2009', 'train/total_time_seconds': '150.6', 'train/time_per_step_avg': '0.3071', 'train/epoch_time_elapsed': '177.2', 'train/estimated_remaining_minutes': '0.438'}
+{'loss': '8.634', 'grad_norm': '1.871e+09', 'learning_rate': '9e-05', 'epoch': '0.2022', 'train/total_time_seconds': '156.6', 'train/time_per_step_avg': '0.3082', 'train/epoch_time_elapsed': '184.2', 'train/estimated_remaining_minutes': '0.4349'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 18.00it/s]
+ 89%|████████▊ | 3100/3500 [03:41<02:16,  2.93it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.635', 'grad_norm': '1.904e+09', 'learning_rate': '9e-05', 'epoch': '0.2036', 'train/total_time_seconds': '163.8', 'train/time_per_step_avg': '0.3128', 'train/epoch_time_elapsed': '192.7', 'train/estimated_remaining_minutes': '0.4338', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001167', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.8', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001152', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.418', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8438', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2962', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.08008', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.023', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.938', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.938', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.812', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '9.75', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1790', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.000843', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.6172', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.5', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.188', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.5', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '10.69', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '2286', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.6484', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.9102', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '3.562', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.9023', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '3.562', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '4.465', 'train/tensor_act_model_layers_0_mlp/norm': '2286', 'train/tensor_act_model_layers_0_mlp/mean': '0.6484', 'train/tensor_act_model_layers_0_mlp/std': '0.9102', 'train/tensor_act_model_layers_0_mlp/max_abs': '3.562', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.9023', 'train/tensor_act_model_layers_0_mlp/max': '3.562', 'train/tensor_act_model_layers_0_mlp/range': '4.465', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '2302', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.6484', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.918', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '3.609', 'train/tensor_act_mode
+{'loss': '8.64', 'grad_norm': '1.913e+09', 'learning_rate': '9e-05', 'epoch': '0.2049', 'train/total_time_seconds': '169.7', 'train/time_per_step_avg': '0.3157', 'train/epoch_time_elapsed': '199.7', 'train/estimated_remaining_minutes': '0.428'}
+{'loss': '8.641', 'grad_norm': '1.929e+09', 'learning_rate': '9e-05', 'epoch': '0.2063', 'train/total_time_seconds': '176.3', 'train/time_per_step_avg': '0.3147', 'train/epoch_time_elapsed': '207.4', 'train/estimated_remaining_minutes': '0.4225', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.7', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001274', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2754', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5547', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001251', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4316', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4316', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3926', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8242', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '3076', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.07324', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.062', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '5.062', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.062', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.812', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '9.875', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1999', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.0003624', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.6914', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.719', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.594', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.719', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.31', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '3112', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.9062', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '1.219', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '4.438', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.8594', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '4.438', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '5.297', 'train/tensor_act_model_layers_0_mlp/norm': '3112', 'train/tensor_act_model_layers_0_mlp/mean': '0.9062', 'train/tensor_act_model_layers_0_mlp/std': '1.219', 'train/tensor_act_model_layers_0_mlp/max_abs': '4.438', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.8594', 'train/tensor_act_model_layers_0_mlp/max': '4.438', 'train/tensor_act_model_layers_0_mlp/range': '5.297', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '3125', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.9102', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '1.227', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.5', 'train/tensor_act_
+{'loss': '8.64', 'grad_norm': '1.963e+09', 'learning_rate': '9e-05', 'epoch': '0.2076', 'train/total_time_seconds': '182.7', 'train/time_per_step_avg': '0.3205', 'train/epoch_time_elapsed': '214.8', 'train/estimated_remaining_minutes': '0.4152'}
+{'loss': '8.642', 'grad_norm': '1.98e+09', 'learning_rate': '9e-05', 'epoch': '0.209', 'train/total_time_seconds': '188.4', 'train/time_per_step_avg': '0.3186', 'train/epoch_time_elapsed': '221.6', 'train/estimated_remaining_minutes': '0.4052'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.92it/s]
+ 91%|█████████▏| 3200/3500 [04:17<01:41,  2.95it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.642', 'grad_norm': '2.022e+09', 'learning_rate': '9e-05', 'epoch': '0.2103', 'train/total_time_seconds': '195.1', 'train/time_per_step_avg': '0.3132', 'train/epoch_time_elapsed': '229.5', 'train/estimated_remaining_minutes': '0.396', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.8', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001183', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2656', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5488', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001228', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3828', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8086', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '3152', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.05542', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.086', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '5.281', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.281', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.812', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '10.09', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '2182', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.002289', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.7539', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.875', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.75', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.875', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.62', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '3886', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '1.133', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '1.523', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '5.375', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.9141', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '5.375', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '6.289', 'train/tensor_act_model_layers_0_mlp/norm': '3886', 'train/tensor_act_model_layers_0_mlp/mean': '1.133', 'train/tensor_act_model_layers_0_mlp/std': '1.523', 'train/tensor_act_model_layers_0_mlp/max_abs': '5.375', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.9141', 'train/tensor_act_model_layers_0_mlp/max': '5.375', 'train/tensor_act_model_layers_0_mlp/range': '6.289', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '3897', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '1.141', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '1.523', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '5.406', 'train/tensor_act_mode
+{'loss': '8.641', 'grad_norm': '2.03e+09', 'learning_rate': '9e-05', 'epoch': '0.2117', 'train/total_time_seconds': '200.7', 'train/time_per_step_avg': '0.3102', 'train/epoch_time_elapsed': '236.2', 'train/estimated_remaining_minutes': '0.3836'}
+{'loss': '8.635', 'grad_norm': '2.013e+09', 'learning_rate': '9e-05', 'epoch': '0.213', 'train/total_time_seconds': '207.4', 'train/time_per_step_avg': '0.3112', 'train/epoch_time_elapsed': '244', 'train/estimated_remaining_minutes': '0.3719', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '187.8', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001183', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2793', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5625', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.6', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001366', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.377', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8027', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '3254', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.04736', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.125', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '5.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.344', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.906', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '10.25', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '2363', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.003601', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.8164', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.938', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.812', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.938', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.75', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '4667', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '1.375', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '1.812', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '6.375', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.9531', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '6.375', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '7.328', 'train/tensor_act_model_layers_0_mlp/norm': '4667', 'train/tensor_act_model_layers_0_mlp/mean': '1.375', 'train/tensor_act_model_layers_0_mlp/std': '1.812', 'train/tensor_act_model_layers_0_mlp/max_abs': '6.375', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.9531', 'train/tensor_act_model_layers_0_mlp/max': '6.375', 'train/tensor_act_model_layers_0_mlp/range': '7.328', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '4678', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '1.383', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '1.82', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '6.5', 'train/tensor_act_model_lay
+{'loss': '8.637', 'grad_norm': '2.038e+09', 'learning_rate': '9e-05', 'epoch': '0.2144', 'train/total_time_seconds': '213.1', 'train/time_per_step_avg': '0.3043', 'train/epoch_time_elapsed': '250.8', 'train/estimated_remaining_minutes': '0.3574'}
+{'loss': '8.642', 'grad_norm': '2.038e+09', 'learning_rate': '9e-05', 'epoch': '0.2157', 'train/total_time_seconds': '218.9', 'train/time_per_step_avg': '0.3049', 'train/epoch_time_elapsed': '257.6', 'train/estimated_remaining_minutes': '0.342'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 14.13it/s]
+ 93%|█████████▎| 3257/3500 [04:39<01:40,  2.41it/s], ?it/s]
+{'loss': '8.636', 'grad_norm': '2.097e+09', 'learning_rate': '9e-05', 'epoch': '0.2171', 'train/total_time_seconds': '225.9', 'train/time_per_step_avg': '0.308', 'train/epoch_time_elapsed': '265.8', 'train/estimated_remaining_minutes': '0.3273', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '188', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.00116', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.2656', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2832', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5488', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '190.9', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.001183', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09326', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.4609', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.4258', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.4609', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.8867', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '3337', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '-0.03711', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.148', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '5.5', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.5', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '5.312', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '10.81', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '2511', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '-0.005219', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.8672', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.938', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.906', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '5.938', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '11.84', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '5459', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '1.625', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '2.109', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '7.469', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-1.008', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '7.469', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '8.477', 'train/tensor_act_model_layers_0_mlp/norm': '5459', 'train/tensor_act_model_layers_0_mlp/mean': '1.625', 'train/tensor_act_model_layers_0_mlp/std': '2.109', 'train/tensor_act_model_layers_0_mlp/max_abs': '7.469', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-1.008', 'train/tensor_act_model_layers_0_mlp/max': '7.469', 'train/tensor_act_model_layers_0_mlp/range': '8.477', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '5468', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '1.625', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '2.125', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '7.594', 'train/tensor_act_model_layers
+{'loss': '8.637', 'grad_norm': '2.097e+09', 'learning_rate': '9e-05', 'epoch': '0.2184', 'train/total_time_seconds': '231.8', 'train/time_per_step_avg': '0.3103', 'train/epoch_time_elapsed': '272.8', 'train/estimated_remaining_minutes': '0.31'}

zain/Activation/wandb/run-20260814_211507-54wnyexz/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260814_211507-54wnyexz/files/wandb-metadata.json

@@ -0,0 +1,99 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-14T21:15:07.166881Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100l.yaml",
+    "--variants",
+    "glu-relu",
+    "glu-gelu",
+    "glu-sigmoid",
+    "glu-waleed10",
+    "glu-silu-waleed10"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "3fb47c6943d6927eafc39bd399a3d3a520bae74a"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "703915499520"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "uv0hmrczrs2tjhgzcbqlzz1jsid4scka"
+}

zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug-core.log

@@ -0,0 +1,63 @@
+{"time":"2026-08-14T19:38:51.636058907Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp54729boe/port-1375539.txt","pid":1375539,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-14T19:38:51.639970993Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1375539}
+{"time":"2026-08-14T19:38:51.639940209Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1375539-170701-251437063/socket","Net":"unix"}}
+{"time":"2026-08-14T19:38:51.812900736Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-14T19:39:07.276688434Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"2(@)"}
+{"time":"2026-08-14T19:39:07.370077387Z","level":"INFO","msg":"handleInformInit: received","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:39:07.640206016Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:39:13.074249733Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"zj3lg1k64oen"}
+{"time":"2026-08-14T19:51:49.732364673Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"zj3lg1k64oen"}
+{"time":"2026-08-14T19:51:50.223490459Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"zj3lg1k64oen"}
+{"time":"2026-08-14T19:51:50.225275352Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:51:50.225791347Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"kk1xbqht","id":"2(@)"}
+{"time":"2026-08-14T19:51:51.140595835Z","level":"INFO","msg":"handleInformInit: received","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T19:51:51.407779103Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T19:51:56.793135206Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"m2u1nqrk4r4j"}
+{"time":"2026-08-14T20:05:01.389026301Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"m2u1nqrk4r4j"}
+{"time":"2026-08-14T20:05:01.891785946Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"m2u1nqrk4r4j"}
+{"time":"2026-08-14T20:05:01.893356932Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T20:05:01.894067235Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ix6wcbwv","id":"2(@)"}
+{"time":"2026-08-14T20:05:03.038763184Z","level":"INFO","msg":"handleInformInit: received","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:05:03.299131712Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:05:08.644597551Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"u5vxhmn1tauv"}
+{"time":"2026-08-14T20:21:26.598738734Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"u5vxhmn1tauv"}
+{"time":"2026-08-14T20:21:27.243676424Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"u5vxhmn1tauv"}
+{"time":"2026-08-14T20:21:27.245262581Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:21:27.245859065Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"bdhnr27j","id":"2(@)"}
+{"time":"2026-08-14T20:21:28.155191217Z","level":"INFO","msg":"handleInformInit: received","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:21:28.414618766Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:21:33.76572887Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"smlkpmpk76rp"}
+{"time":"2026-08-14T20:38:01.401128169Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"smlkpmpk76rp"}
+{"time":"2026-08-14T20:38:02.166866415Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"smlkpmpk76rp"}
+{"time":"2026-08-14T20:38:02.168373578Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:38:02.169204491Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"twgo1e89","id":"2(@)"}
+{"time":"2026-08-14T20:38:03.474631884Z","level":"INFO","msg":"handleInformInit: received","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:38:03.739404113Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:38:09.1668666Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"9gr96sam7ydd"}
+{"time":"2026-08-14T20:54:40.077063626Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"9gr96sam7ydd"}
+{"time":"2026-08-14T20:54:40.77979146Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"9gr96sam7ydd"}
+{"time":"2026-08-14T20:54:40.781496498Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:54:40.782279361Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"7p5140bq","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.400855251Z","level":"INFO","msg":"connection: closing","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.40094641Z","level":"INFO","msg":"connection: closed successfully","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.400864953Z","level":"INFO","msg":"processOutgoingData: finished","id":"2(@)"}
+{"time":"2026-08-14T20:54:42.400960827Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"2(@)"}
+{"time":"2026-08-14T21:01:09.497626422Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"3(@)"}
+{"time":"2026-08-14T21:01:09.714752804Z","level":"INFO","msg":"handleInformInit: received","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:01:09.994191533Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:01:15.507216386Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"wd49dhcig506"}
+{"time":"2026-08-14T21:07:53.011770412Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"wd49dhcig506"}
+{"time":"2026-08-14T21:07:54.661515353Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"wd49dhcig506"}
+{"time":"2026-08-14T21:07:54.926288764Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:07:54.927423677Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"qauftiuh","id":"3(@)"}
+{"time":"2026-08-14T21:08:09.037499672Z","level":"INFO","msg":"handleInformInit: received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:08:09.296770198Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:08:09.806890437Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"bsqpd0c29zk9"}
+{"time":"2026-08-14T21:08:14.888124439Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:51.274587185Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.85240857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"ilk9hnwtzxxz"}
+{"time":"2026-08-14T21:14:52.873645397Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:14:52.874311484Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"a6574zx8","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.169254211Z","level":"INFO","msg":"handleInformInit: received","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:07.435699541Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"54wnyexz","id":"3(@)"}
+{"time":"2026-08-14T21:15:12.853675736Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"fzqy9cymfofj"}

zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug-internal.log

@@ -0,0 +1,65 @@
+{"time":"2026-08-14T21:15:07.169424866Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-14T21:15:07.169567468Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-14T21:15:07.435532929Z","level":"INFO","msg":"stream: created new stream","id":"54wnyexz"}
+{"time":"2026-08-14T21:15:07.435601691Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-14T21:15:07.435689178Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-14T21:15:07.43571596Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-14T21:15:07.435698857Z","level":"INFO","msg":"writer: started","stream_id":"54wnyexz"}
+{"time":"2026-08-14T21:15:07.865306518Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":2}
+{"time":"2026-08-14T21:15:07.979243259Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:22.8655656Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-14T21:15:23.014629997Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:23.378577725Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":1538}
+{"time":"2026-08-14T21:15:23.378611071Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-08-14T21:15:23.38845317Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":4102}
+{"time":"2026-08-14T21:15:23.388483609Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-08-14T21:15:23.389724048Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":4603}
+{"time":"2026-08-14T21:15:23.389898245Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":36}
+{"time":"2026-08-14T21:15:23.39094459Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":5037}
+{"time":"2026-08-14T21:15:23.391389079Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":103}
+{"time":"2026-08-14T21:15:23.391844857Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":5345}
+{"time":"2026-08-14T21:15:23.398774475Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":2645}
+{"time":"2026-08-14T21:15:23.399288691Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":8229}
+{"time":"2026-08-14T21:15:23.400206063Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":379}
+{"time":"2026-08-14T21:15:23.400644516Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":8832}
+{"time":"2026-08-14T21:15:23.400737823Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":19}
+{"time":"2026-08-14T21:15:23.400946664Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":8941}
+{"time":"2026-08-14T21:15:23.401087444Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":37}
+{"time":"2026-08-14T21:15:23.405809502Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":10867}
+{"time":"2026-08-14T21:15:23.405942913Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":22}
+{"time":"2026-08-14T21:15:23.407538574Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":11526}
+{"time":"2026-08-14T21:15:23.408103184Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":213}
+{"time":"2026-08-14T21:15:37.883634237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":2,"events_offset":1,"events_lines":2,"console_offset":1,"console_lines":1}
+{"time":"2026-08-14T21:15:38.603529933Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:15:52.884551759Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":3,"events_offset":3,"events_lines":2,"console_offset":1,"console_lines":1}
+{"time":"2026-08-14T21:15:53.827374416Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:07.891440937Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":2,"events_offset":5,"events_lines":2,"console_offset":3,"console_lines":12}
+{"time":"2026-08-14T21:16:08.876461742Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:22.890274165Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":2,"events_offset":7,"events_lines":2,"console_offset":11,"console_lines":1}
+{"time":"2026-08-14T21:16:23.704159399Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:37.88767292Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":2,"events_offset":9,"events_lines":2,"console_offset":15,"console_lines":9}
+{"time":"2026-08-14T21:16:38.690316648Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:16:52.886427045Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":2,"events_offset":11,"events_lines":2,"console_offset":21,"console_lines":1}
+{"time":"2026-08-14T21:16:54.006407739Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:07.874889179Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":2,"events_offset":13,"events_lines":2,"console_offset":21,"console_lines":1}
+{"time":"2026-08-14T21:17:08.765471891Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:22.884402781Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":15,"events_lines":2,"console_offset":24,"console_lines":11}
+{"time":"2026-08-14T21:17:23.782618431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:37.894674616Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":2,"events_offset":17,"events_lines":2,"console_offset":31,"console_lines":1}
+{"time":"2026-08-14T21:17:38.741283006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:17:52.887547658Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":2,"events_offset":19,"events_lines":2,"console_offset":35,"console_lines":9}
+{"time":"2026-08-14T21:17:53.759903107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:07.894710599Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":2,"events_offset":21,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-08-14T21:18:08.817504637Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:22.875702416Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":2,"events_offset":23,"events_lines":2,"console_offset":41,"console_lines":1}
+{"time":"2026-08-14T21:18:23.642998715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:37.887794344Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":2,"events_offset":25,"events_lines":2,"console_offset":44,"console_lines":11}
+{"time":"2026-08-14T21:18:38.696615712Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:18:52.883505832Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":2,"events_offset":27,"events_lines":2,"console_offset":51,"console_lines":1}
+{"time":"2026-08-14T21:18:53.743917259Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:07.888490889Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":2,"events_offset":29,"events_lines":2,"console_offset":55,"console_lines":9}
+{"time":"2026-08-14T21:19:08.707657114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:22.884734385Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":2,"events_offset":31,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-08-14T21:19:23.790343089Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:19:37.875793912Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":2,"events_offset":33,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-08-14T21:19:38.585616478Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug.log

@@ -0,0 +1,20 @@
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug.log
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug-internal.log
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():772] calling init triggers
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():820] starting backend
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-14 21:15:07,168 INFO    MainThread:936813 [wandb_init.py:init():835] sending inform_init request
+2026-08-14 21:15:07,436 INFO    MainThread:936813 [wandb_init.py:init():840] backend started and connected
+2026-08-14 21:15:07,437 INFO    MainThread:936813 [wandb_init.py:init():910] updated telemetry
+2026-08-14 21:15:07,444 INFO    MainThread:936813 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-14 21:15:07,772 INFO    MainThread:936813 [wandb_init.py:init():978] starting run threads in backend
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-14 21:15:07,851 INFO    MainThread:936813 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-14 21:15:07,852 INFO    MainThread:936813 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-14 21:15:07,853 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 100, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'sigmoid', 'waleed_beta': 4, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-sigmoid-100L_trash_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 3500, 'learning_rate': 9e-05, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 50, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-sigmoid-100L-16.9M-20260814-211506', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 2498, 'eval_delay': 0, 'per_device_eval_batch_size': 1024, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 50, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-sigmoid-100L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-14 21:15:07,858 INFO    MainThread:936813 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 16934016 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee39eebe10>>
+2026-08-14 21:15:07,858 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 16934016 None

zain/Activation/wandb/run-20260814_211507-54wnyexz/run-54wnyexz.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94d0301065b1dbd1d98ffb4d98d6bb8e62964643c148bc6e9e326f26b94cae64
+size 37814272