Auto upload zain 2026-08-14T21:35:16.328459 (part 4)

zain/Activation/out/sweep_summary.json

@@ -1,51 +1,37 @@
 [
   {
-    "variant": "glu-waleed10-9L",
-    "eval_loss": 2.248227596282959,
-    "out": "out/glu-waleed10-9L_run",
-    "run_name": "LM-glu-waleed10-9L-2.0M-20260814-200424",
+    "variant": "glu-relu-100L",
+    "eval_loss": 8.116716384887695,
+    "out": "out/glu-relu-100L_trash_run",
+    "run_name": "LM-glu-relu-100L-16.9M-20260814-210108",
     "status": "success"
   },
   {
-    "variant": "glu-silu-waleed10-9L",
-    "eval_loss": 2.241689443588257,
-    "out": "out/glu-silu-waleed10-9L_run",
-    "run_name": "LM-glu-silu-waleed10-9L-2.0M-20260814-201615",
+    "variant": "glu-gelu-100L",
+    "eval_loss": 8.100676536560059,
+    "out": "out/glu-gelu-100L_trash_run",
+    "run_name": "LM-glu-gelu-100L-16.9M-20260814-210807",
     "status": "success"
   },
   {
-    "variant": "mlp-waleed10-9L",
-    "eval_loss": 2.6366770267486572,
-    "out": "out/mlp-waleed10-9L_run",
-    "run_name": "LM-mlp-waleed10-9L-2.0M-20260814-202806",
+    "variant": "glu-sigmoid-100L",
+    "eval_loss": 8.628765106201172,
+    "out": "out/glu-sigmoid-100L_trash_run",
+    "run_name": "LM-glu-sigmoid-100L-16.9M-20260814-211506",
     "status": "success"
   },
   {
-    "variant": "mlp-silu-waleed10-9L",
-    "eval_loss": 2.4668548107147217,
-    "out": "out/mlp-silu-waleed10-9L_run",
-    "run_name": "LM-mlp-silu-waleed10-9L-2.0M-20260814-203944",
+    "variant": "glu-waleed10-100L",
+    "eval_loss": 7.94517707824707,
+    "out": "out/glu-waleed10-100L_trash_run",
+    "run_name": "LM-glu-waleed10-100L-16.9M-20260814-212207",
     "status": "success"
   },
   {
-    "variant": "glu-waleed-9L",
-    "eval_loss": 2.2454068660736084,
-    "out": "out/glu-waleed-9L_run",
-    "run_name": "LM-glu-waleed-9L-2.0M-20260814-205134",
-    "status": "success"
-  },
-  {
-    "variant": "glu-situglu_low-9L",
-    "eval_loss": 2.262706995010376,
-    "out": "out/glu-situglu_low-9L_run",
-    "run_name": "LM-glu-situglu_low-9L-2.0M-20260814-210155",
-    "status": "success"
-  },
-  {
-    "variant": "glu-waleedglu_low-9L",
-    "eval_loss": 2.23911452293396,
-    "out": "out/glu-waleedglu_low-9L_run",
-    "run_name": "LM-glu-waleedglu_low-9L-2.0M-20260814-211407",
+    "variant": "glu-silu-waleed10-100L",
+    "eval_loss": 8.10374927520752,
+    "out": "out/glu-silu-waleed10-100L_trash_run",
+    "run_name": "LM-glu-silu-waleed10-100L-16.9M-20260814-212834",
     "status": "success"
   }
 ]

zain/Activation/wandb/debug-internal.log

@@ -1,47 +1,37 @@
-{"time":"2026-08-14T21:28:35.344927311Z","level":"INFO","msg":"wandb-core"}
-{"time":"2026-08-14T21:28:35.34506265Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
-{"time":"2026-08-14T21:28:35.602855263Z","level":"INFO","msg":"stream: created new stream","id":"gqskne0x"}
-{"time":"2026-08-14T21:28:35.602933409Z","level":"INFO","msg":"handler: started"}
-{"time":"2026-08-14T21:28:35.603016258Z","level":"INFO","msg":"stream: started"}
-{"time":"2026-08-14T21:28:35.603043817Z","level":"INFO","msg":"writer: started","stream_id":"gqskne0x"}
-{"time":"2026-08-14T21:28:35.60305765Z","level":"INFO","msg":"sender: started"}
-{"time":"2026-08-14T21:28:36.090276496Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
-{"time":"2026-08-14T21:28:36.211652827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:28:51.09126529Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":3,"uploaded_len":2}
-{"time":"2026-08-14T21:28:51.261717283Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:28:53.048390257Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":3104}
-{"time":"2026-08-14T21:28:53.048417504Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
-{"time":"2026-08-14T21:28:53.050441969Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":3665}
-{"time":"2026-08-14T21:28:53.050479366Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
-{"time":"2026-08-14T21:28:53.051308024Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":3981}
-{"time":"2026-08-14T21:28:53.052248742Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":269}
-{"time":"2026-08-14T21:28:53.052348345Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":4293}
-{"time":"2026-08-14T21:28:53.053710973Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":419}
-{"time":"2026-08-14T21:28:53.054973484Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":5282}
-{"time":"2026-08-14T21:28:53.05654974Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":576}
-{"time":"2026-08-14T21:28:53.057424589Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":6166}
-{"time":"2026-08-14T21:28:53.058161238Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":225}
-{"time":"2026-08-14T21:28:53.059182778Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":6765}
-{"time":"2026-08-14T21:28:53.059372384Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":41}
-{"time":"2026-08-14T21:28:53.059719473Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":6917}
-{"time":"2026-08-14T21:28:53.059850702Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":21}
-{"time":"2026-08-14T21:28:53.061202591Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":7342}
-{"time":"2026-08-14T21:28:53.061276823Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":12}
-{"time":"2026-08-14T21:28:53.068366857Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":9009}
-{"time":"2026-08-14T21:28:53.068403516Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
-{"time":"2026-08-14T21:28:53.070211602Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":9602}
-{"time":"2026-08-14T21:28:53.071216814Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":294}
-{"time":"2026-08-14T21:28:53.071521294Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":10000}
-{"time":"2026-08-14T21:28:53.072106936Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":190}
-{"time":"2026-08-14T21:28:53.075408887Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":11406}
-{"time":"2026-08-14T21:28:53.076614643Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":333}
-{"time":"2026-08-14T21:29:06.110603465Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":2,"events_offset":1,"events_lines":2,"console_offset":1,"console_lines":1}
-{"time":"2026-08-14T21:29:06.83621196Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:29:21.108433332Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":3,"events_offset":3,"events_lines":2,"console_offset":1,"console_lines":1}
-{"time":"2026-08-14T21:29:21.999015678Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:29:36.110784946Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":2,"events_offset":5,"events_lines":2,"console_offset":3,"console_lines":12}
-{"time":"2026-08-14T21:29:36.946170207Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:29:51.109458297Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":3,"events_offset":7,"events_lines":2,"console_offset":11,"console_lines":1}
-{"time":"2026-08-14T21:29:52.226871571Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
-{"time":"2026-08-14T21:30:06.110393089Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":2,"events_offset":9,"events_lines":2,"console_offset":15,"console_lines":10}
-{"time":"2026-08-14T21:30:07.570174219Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:31:53.802248642Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-14T21:31:53.802536305Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-14T21:31:54.062231678Z","level":"INFO","msg":"stream: created new stream","id":"q9vcmje5"}
+{"time":"2026-08-14T21:31:54.062319994Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-14T21:31:54.062390309Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-14T21:31:54.062412718Z","level":"INFO","msg":"writer: started","stream_id":"q9vcmje5"}
+{"time":"2026-08-14T21:31:54.062420181Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-14T21:31:54.416838412Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-14T21:31:54.510348438Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:32:09.41771802Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":3,"uploaded_len":2}
+{"time":"2026-08-14T21:32:09.578280298Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:32:24.417605975Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":1,"events_lines":2,"console_offset":2,"console_lines":2}
+{"time":"2026-08-14T21:32:24.549935438Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:32:39.41719538Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":3,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-14T21:32:39.566802488Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:32:54.417058222Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-14T21:32:54.526885785Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:33:09.417361811Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":7,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-14T21:33:09.541292122Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:33:24.41766197Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":9,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-14T21:33:24.549198743Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:33:39.417110044Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":11,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-14T21:33:39.551776959Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:33:54.417538658Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":13,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-14T21:33:54.555356936Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:34:09.417890895Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":15,"events_lines":2,"console_offset":4,"console_lines":10}
+{"time":"2026-08-14T21:34:09.592684206Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:34:24.417179414Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":17,"events_lines":2,"console_offset":12,"console_lines":1}
+{"time":"2026-08-14T21:34:24.534710617Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:34:39.417073445Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":19,"events_lines":2,"console_offset":12,"console_lines":1}
+{"time":"2026-08-14T21:34:39.570434803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:34:54.417818476Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":21,"events_lines":2,"console_offset":12,"console_lines":1}
+{"time":"2026-08-14T21:34:54.541208219Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:35:09.41711858Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":23,"events_lines":2,"console_offset":12,"console_lines":1}
+{"time":"2026-08-14T21:35:09.54503893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-14T21:35:24.417709905Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":25,"events_lines":2,"console_offset":12,"console_lines":1}
+{"time":"2026-08-14T21:35:24.538800467Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/debug.log

@@ -1,20 +1,22 @@
-2026-08-14 21:28:35,343 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_212835-gqskne0x/logs/debug.log
-2026-08-14 21:28:35,343 INFO    MainThread:936813 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_212835-gqskne0x/logs/debug-internal.log
-2026-08-14 21:28:35,343 INFO    MainThread:936813 [wandb_init.py:init():772] calling init triggers
-2026-08-14 21:28:35,343 INFO    MainThread:936813 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+2026-08-14 21:31:53,800 INFO    MainThread:1229423 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
+2026-08-14 21:31:53,800 INFO    MainThread:1229423 [wandb_setup.py:_flush():81] Configure stats pid to 1229423
+2026-08-14 21:31:53,800 INFO    MainThread:1229423 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-14 21:31:53,800 INFO    MainThread:1229423 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_213153-q9vcmje5/logs/debug.log
+2026-08-14 21:31:53,800 INFO    MainThread:1229423 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260814_213153-q9vcmje5/logs/debug-internal.log
+2026-08-14 21:31:53,800 INFO    MainThread:1229423 [wandb_init.py:init():772] calling init triggers
+2026-08-14 21:31:53,801 INFO    MainThread:1229423 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2026-08-14 21:28:35,343 INFO    MainThread:936813 [wandb_init.py:init():820] starting backend
-2026-08-14 21:28:35,343 INFO    MainThread:936813 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
-2026-08-14 21:28:35,343 INFO    MainThread:936813 [wandb_init.py:init():835] sending inform_init request
-2026-08-14 21:28:35,603 INFO    MainThread:936813 [wandb_init.py:init():840] backend started and connected
-2026-08-14 21:28:35,605 INFO    MainThread:936813 [wandb_init.py:init():910] updated telemetry
-2026-08-14 21:28:35,614 INFO    MainThread:936813 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
-2026-08-14 21:28:35,996 INFO    MainThread:936813 [wandb_init.py:init():978] starting run threads in backend
-2026-08-14 21:28:36,075 INFO    MainThread:936813 [wandb_run.py:_console_start():2621] atexit reg
-2026-08-14 21:28:36,075 INFO    MainThread:936813 [wandb_run.py:_redirect():2471] redirect: wrap_raw
-2026-08-14 21:28:36,076 INFO    MainThread:936813 [wandb_run.py:_redirect():2540] Wrapping output streams.
-2026-08-14 21:28:36,076 INFO    MainThread:936813 [wandb_run.py:_redirect():2563] Redirects installed.
-2026-08-14 21:28:36,077 INFO    MainThread:936813 [wandb_init.py:init():1016] run started, returning control to user process
-2026-08-14 21:28:36,078 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 100, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'silu-waleed10', 'waleed_beta': 4, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-silu-waleed10-100L_trash_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 3500, 'learning_rate': 9e-05, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 50, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-silu-waleed10-100L-16.9M-20260814-212834', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 2498, 'eval_delay': 0, 'per_device_eval_batch_size': 1024, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 50, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-silu-waleed10-100L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
-2026-08-14 21:28:36,083 INFO    MainThread:936813 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 16934016 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14ee399dda90>>
-2026-08-14 21:28:36,083 INFO    MainThread:936813 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 16934016 None
+2026-08-14 21:31:53,801 INFO    MainThread:1229423 [wandb_init.py:init():820] starting backend
+2026-08-14 21:31:53,801 INFO    MainThread:1229423 [wandb_init.py:init():835] sending inform_init request
+2026-08-14 21:31:54,062 INFO    MainThread:1229423 [wandb_init.py:init():840] backend started and connected
+2026-08-14 21:31:54,063 INFO    MainThread:1229423 [wandb_init.py:init():910] updated telemetry
+2026-08-14 21:31:54,070 INFO    MainThread:1229423 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-14 21:31:54,336 INFO    MainThread:1229423 [wandb_init.py:init():978] starting run threads in backend
+2026-08-14 21:31:54,409 INFO    MainThread:1229423 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-14 21:31:54,409 INFO    MainThread:1229423 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-14 21:31:54,409 INFO    MainThread:1229423 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-14 21:31:54,409 INFO    MainThread:1229423 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-14 21:31:54,412 INFO    MainThread:1229423 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-14 21:31:54,413 INFO    MainThread:1229423 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 21, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'waleed10', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-waleed10-21L_run', 'per_device_train_batch_size': 128, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.001, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': None, 'warmup_steps': 0, 'optim': 'adamw_torch', 'optim_args': None, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 4, 'average_tokens_across_devices': True, 'max_grad_norm': 1.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-waleed10-21L-4.0M-20260814-213152', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 60000, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 100, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/A-glu-waleed10-21L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-14 21:31:54,415 INFO    MainThread:1229423 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 3970432 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x1528c060ecd0>>
+2026-08-14 21:31:54,415 INFO    MainThread:1229423 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 3970432 None

zain/Activation/wandb/run-20260814_193907-kk1xbqht/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_195151-ix6wcbwv/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_200503-bdhnr27j/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_202128-twgo1e89/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_203803-7p5140bq/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_210109-qauftiuh/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_210809-a6574zx8/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_211507-54wnyexz/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_212208-n6fjpg9o/logs/debug-core.log

@@ -75,3 +75,11 @@
 {"time":"2026-08-14T21:28:35.344777056Z","level":"INFO","msg":"handleInformInit: received","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:35.603024157Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"gqskne0x","id":"3(@)"}
 {"time":"2026-08-14T21:28:41.078893683Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:05.007052341Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.745707472Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"yxbo55z35lse"}
+{"time":"2026-08-14T21:35:06.767299957Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:06.773662133Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"gqskne0x","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806158062Z","level":"INFO","msg":"connection: closing","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806241813Z","level":"INFO","msg":"connection: closed successfully","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806161779Z","level":"INFO","msg":"processOutgoingData: finished","id":"3(@)"}
+{"time":"2026-08-14T21:35:08.806251477Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"3(@)"}

zain/Activation/wandb/run-20260814_212835-gqskne0x/files/config.yaml

@@ -0,0 +1,439 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            pf2h3cdzqvkrfjeqx7v0rn4hp1f52wde:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline100l.yaml
+                    - --variants
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-waleed10
+                    - glu-silu-waleed10
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "709055246336"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: 3fb47c6943d6927eafc39bd399a3d3a520bae74a
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-14T21:28:35.342440Z"
+                writerId: pf2h3cdzqvkrfjeqx7v0rn4hp1f52wde
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 41
+                - 61
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: silu-waleed10
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 2498
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-glu-silu-waleed10-100L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 9e-05
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 1
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 3500
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 16934016
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 100
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-silu-waleed10-100L_trash_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 1024
+per_device_train_batch_size:
+    value: 64
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-silu-waleed10-100L-16.9M-20260814-212834
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 50
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 4
+warmup_steps:
+    value: 50
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260814_212835-gqskne0x/files/output.log

@@ -19,8 +19,93 @@ Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.
   - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
   - If you are not the owner of the model architecture class, please contact the model code owner to update it.
 Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 13.82it/s]
- 80%|███████▉  | 2797/3500 [01:34<03:16,  3.57it/s], ?it/s]
+ 80%|████████  | 2800/3500 [01:35<03:15,  3.59it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
 {'loss': '8.051', 'grad_norm': '3.817e+08', 'learning_rate': '9e-05', 'epoch': '0.1834', 'train/total_time_seconds': '60.56', 'train/time_per_step_avg': '0.2375', 'train/epoch_time_elapsed': '72.75', 'train/estimated_remaining_minutes': '0.2894', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009193', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '-1.162e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3555', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3555', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3047', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6602', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1488', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.0188', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.5117', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '3.141', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-3.141', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '2.406', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '5.547', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1454', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.006104', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.5039', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '2.469', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-2.469', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '2.469', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '4.938', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '492', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.125', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.2051', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '1.398', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.7461', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '1.398', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '2.145', 'train/tensor_act_model_layers_0_mlp/norm': '487.8', 'train/tensor_act_model_layers_0_mlp/mean': '0.1245', 'train/tensor_act_model_layers_0_mlp/std': '0.2031', 'train/tensor_act_model_layers_0_mlp/max_abs': '1.344', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.7383', 'train/tensor_act_model_layers_0_mlp/max': '1.344', 'train/tensor_act_model_layers_0_mlp/range': '2.082', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '542.1', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.1245', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.2334', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '1.406', 'train/tenso
 {'loss': '8.067', 'grad_norm': '3.02e+08', 'learning_rate': '9e-05', 'epoch': '0.1847', 'train/total_time_seconds': '65.12', 'train/time_per_step_avg': '0.2378', 'train/epoch_time_elapsed': '78.35', 'train/estimated_remaining_minutes': '0.301'}
 {'loss': '8.077', 'grad_norm': '2.317e+08', 'learning_rate': '9e-05', 'epoch': '0.186', 'train/total_time_seconds': '70.12', 'train/time_per_step_avg': '0.2378', 'train/epoch_time_elapsed': '84.44', 'train/estimated_remaining_minutes': '0.3133', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186.1', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0008926', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.2', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '-3.07e-06', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3262', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3262', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.2949', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6211', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '1897', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.03638', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.6523', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '4.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-4.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '3.609', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '8.047', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '1877', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.01044', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.6484', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '3.797', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-3.797', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '3.422', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '7.219', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '2102', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '0.6133', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '0.8242', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '4.688', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.8867', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '4.688', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '5.574', 'train/tensor_act_model_layers_0_mlp/norm': '1878', 'train/tensor_act_model_layers_0_mlp/mean': '0.5703', 'train/tensor_act_model_layers_0_mlp/std': '0.7188', 'train/tensor_act_model_layers_0_mlp/max_abs': '3.297', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.8711', 'train/tensor_act_model_layers_0_mlp/max': '3.297', 'train/tensor_act_model_layers_0_mlp/range': '4.168', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '1897', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '0.5703', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.7305', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '3.422', 'train/tensor_
 {'loss': '8.085', 'grad_norm': '1.646e+08', 'learning_rate': '9e-05', 'epoch': '0.1874', 'train/total_time_seconds': '74.72', 'train/time_per_step_avg': '0.2379', 'train/epoch_time_elapsed': '90.08', 'train/estimated_remaining_minutes': '0.3225'}
+{'loss': '8.09', 'grad_norm': '1.091e+08', 'learning_rate': '9e-05', 'epoch': '0.1887', 'train/total_time_seconds': '79.33', 'train/time_per_step_avg': '0.2381', 'train/epoch_time_elapsed': '95.69', 'train/estimated_remaining_minutes': '0.3305'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.56it/s]
+ 83%|████████▎ | 2900/3500 [02:05<02:48,  3.56it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.097', 'grad_norm': '7.13e+07', 'learning_rate': '9e-05', 'epoch': '0.1901', 'train/total_time_seconds': '84.42', 'train/time_per_step_avg': '0.2386', 'train/epoch_time_elapsed': '102.1', 'train/estimated_remaining_minutes': '0.3393', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009193', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.2', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '-6.437e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3555', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3555', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3242', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6797', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '2680', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.06152', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '0.9219', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '5.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-5.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '4.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '10.25', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '2686', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.01282', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '0.9258', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '5.281', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-5.281', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '4.781', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '10.06', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '8155', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '2.547', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '3.062', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '14.38', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.03122', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.3613', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '14.38', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '14.74', 'train/tensor_act_model_layers_0_mlp/norm': '4345', 'train/tensor_act_model_layers_0_mlp/mean': '1.586', 'train/tensor_act_model_layers_0_mlp/std': '1.414', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.3594', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '4.359', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '4353', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '1.586', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '1.414', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.219', 'train/tensor_act_mod
+{'loss': '8.106', 'grad_norm': '3.185e+07', 'learning_rate': '9e-05', 'epoch': '0.1914', 'train/total_time_seconds': '89.24', 'train/time_per_step_avg': '0.2412', 'train/epoch_time_elapsed': '107.9', 'train/estimated_remaining_minutes': '0.3456'}
+{'loss': '8.104', 'grad_norm': '1.743e+07', 'learning_rate': '9e-05', 'epoch': '0.1928', 'train/total_time_seconds': '94.27', 'train/time_per_step_avg': '0.2415', 'train/epoch_time_elapsed': '114.1', 'train/estimated_remaining_minutes': '0.3516', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186.2', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001076', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.5', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0002203', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.0918', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3281', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3281', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3125', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6406', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '3676', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.06934', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.266', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '6.031', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.031', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '5.938', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '11.97', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '3721', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.01917', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.281', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '6.25', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-6.25', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '6.219', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '12.47', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '2.066e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '6.844', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '7.406', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '32.25', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.2761', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.4883', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '32.25', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '32.74', 'train/tensor_act_model_layers_0_mlp/norm': '5854', 'train/tensor_act_model_layers_0_mlp/mean': '2.469', 'train/tensor_act_model_layers_0_mlp/std': '1.438', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.4863', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '4.486', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '5859', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '2.469', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '1.445', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.281', 'train/tensor_act_model
+{'loss': '8.107', 'grad_norm': '1.226e+07', 'learning_rate': '9e-05', 'epoch': '0.1941', 'train/total_time_seconds': '98.84', 'train/time_per_step_avg': '0.2412', 'train/epoch_time_elapsed': '119.7', 'train/estimated_remaining_minutes': '0.3546'}
+{'loss': '8.105', 'grad_norm': '1.022e+07', 'learning_rate': '9e-05', 'epoch': '0.1955', 'train/total_time_seconds': '103.5', 'train/time_per_step_avg': '0.2413', 'train/epoch_time_elapsed': '125.3', 'train/estimated_remaining_minutes': '0.3568'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 17.45it/s]
+ 86%|████████▌ | 3000/3500 [02:34<02:20,  3.56it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.104', 'grad_norm': '1.016e+07', 'learning_rate': '9e-05', 'epoch': '0.1968', 'train/total_time_seconds': '108.6', 'train/time_per_step_avg': '0.2415', 'train/epoch_time_elapsed': '131.7', 'train/estimated_remaining_minutes': '0.3594', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009613', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '7.629e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.332', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.332', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3105', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6426', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4636', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.08057', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.602', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.219', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.562', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.219', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '13.78', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '4710', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.02734', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.625', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.562', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.562', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.375', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '14.94', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '4.009e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '14.06', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '13.56', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '60', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.4477', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '-0.08105', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '60', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '60.08', 'train/tensor_act_model_layers_0_mlp/norm': '6960', 'train/tensor_act_model_layers_0_mlp/mean': '3.219', 'train/tensor_act_model_layers_0_mlp/std': '1.078', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '-0.08105', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '4.081', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '6963', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.219', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '1.078', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.281', 'train/tensor_act_model_l
+{'loss': '8.105', 'grad_norm': '1.979e+07', 'learning_rate': '9e-05', 'epoch': '0.1982', 'train/total_time_seconds': '113.2', 'train/time_per_step_avg': '0.2395', 'train/epoch_time_elapsed': '137.4', 'train/estimated_remaining_minutes': '0.3593'}
+{'loss': '8.1', 'grad_norm': '6.488e+06', 'learning_rate': '9e-05', 'epoch': '0.1995', 'train/total_time_seconds': '118.3', 'train/time_per_step_avg': '0.2399', 'train/epoch_time_elapsed': '143.6', 'train/estimated_remaining_minutes': '0.3596', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009537', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.2', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '2.337e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3242', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3242', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3125', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6367', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '4988', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.09424', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.188', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.5', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.188', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '13.69', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5069', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.03174', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.75', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.75', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.5', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.75', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.25', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '4.896e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '18.12', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '15.62', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '68.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.5547', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '0.1484', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '68.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '68.35', 'train/tensor_act_model_layers_0_mlp/norm': '7503', 'train/tensor_act_model_layers_0_mlp/mean': '3.594', 'train/tensor_act_model_layers_0_mlp/std': '0.7109', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '0.1484', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '3.852', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '7505', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.594', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.7188', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.281', 'train/tensor_act_model_layers_0
+{'loss': '8.102', 'grad_norm': '4.719e+06', 'learning_rate': '9e-05', 'epoch': '0.2009', 'train/total_time_seconds': '122.9', 'train/time_per_step_avg': '0.2403', 'train/epoch_time_elapsed': '149.2', 'train/estimated_remaining_minutes': '0.3573'}
+{'loss': '8.102', 'grad_norm': '3.654e+06', 'learning_rate': '9e-05', 'epoch': '0.2022', 'train/total_time_seconds': '127.5', 'train/time_per_step_avg': '0.2402', 'train/epoch_time_elapsed': '154.8', 'train/estimated_remaining_minutes': '0.3541'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.23it/s]
+ 89%|████████▊ | 3100/3500 [03:04<01:51,  3.60it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50��� onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.106', 'grad_norm': '2.703e+06', 'learning_rate': '9e-05', 'epoch': '0.2036', 'train/total_time_seconds': '132.5', 'train/time_per_step_avg': '0.2397', 'train/epoch_time_elapsed': '161.2', 'train/estimated_remaining_minutes': '0.3511', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186.1', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.000946', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '6.58e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.332', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6738', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5243', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.1055', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.805', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.375', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.656', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.375', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.03', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5329', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.0332', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.836', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.719', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.5', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.719', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.22', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '5.638e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '21.88', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '16.75', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '83.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.6718', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '0.8672', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '83.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '82.63', 'train/tensor_act_model_layers_0_mlp/norm': '7914', 'train/tensor_act_model_layers_0_mlp/mean': '3.844', 'train/tensor_act_model_layers_0_mlp/std': '0.3535', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '0.8555', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '3.145', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '7916', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.844', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.3672', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.281', 'train/tensor_act_model_laye
+{'loss': '8.108', 'grad_norm': '1.999e+06', 'learning_rate': '9e-05', 'epoch': '0.2049', 'train/total_time_seconds': '137.1', 'train/time_per_step_avg': '0.2394', 'train/epoch_time_elapsed': '166.8', 'train/estimated_remaining_minutes': '0.3458'}
+{'loss': '8.102', 'grad_norm': '1.516e+06', 'learning_rate': '9e-05', 'epoch': '0.2063', 'train/total_time_seconds': '142.2', 'train/time_per_step_avg': '0.239', 'train/epoch_time_elapsed': '172.9', 'train/estimated_remaining_minutes': '0.3407', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.8', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009766', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '7.629e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3184', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3184', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3125', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6309', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5401', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.1074', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.859', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.906', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.78', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5496', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.03369', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.898', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '8.062', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.562', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '8.062', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.62', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.123e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '24.62', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '16.88', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '94', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.8062', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '1.547', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '94', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '92.45', 'train/tensor_act_model_layers_0_mlp/norm': '8089', 'train/tensor_act_model_layers_0_mlp/mean': '3.953', 'train/tensor_act_model_layers_0_mlp/std': '0.1523', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '1.477', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '2.523', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8092', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.953', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1787', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layer
+{'loss': '8.101', 'grad_norm': '1.155e+06', 'learning_rate': '9e-05', 'epoch': '0.2076', 'train/total_time_seconds': '146.8', 'train/time_per_step_avg': '0.239', 'train/epoch_time_elapsed': '178.6', 'train/estimated_remaining_minutes': '0.3336'}
+{'loss': '8.105', 'grad_norm': '8.602e+05', 'learning_rate': '9e-05', 'epoch': '0.209', 'train/total_time_seconds': '151.3', 'train/time_per_step_avg': '0.2385', 'train/epoch_time_elapsed': '184.1', 'train/estimated_remaining_minutes': '0.3254'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.19it/s]
+ 91%|█████████▏| 3200/3500 [03:40<01:51,  2.68it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.107', 'grad_norm': '6.513e+05', 'learning_rate': '9e-05', 'epoch': '0.2103', 'train/total_time_seconds': '156.4', 'train/time_per_step_avg': '0.2383', 'train/epoch_time_elapsed': '190.5', 'train/estimated_remaining_minutes': '0.3174', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009766', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.2', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '-3.91e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.332', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6816', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5413', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.1016', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.867', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.31', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5512', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.03271', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.906', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.938', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.531', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.938', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.47', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.279e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '25.75', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '16.62', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '95.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.8794', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '2.266', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '95.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '93.23', 'train/tensor_act_model_layers_0_mlp/norm': '8143', 'train/tensor_act_model_layers_0_mlp/mean': '3.969', 'train/tensor_act_model_layers_0_mlp/std': '0.0752', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '2.047', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '1.953', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8145', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.969', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.1187', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.344', 'train/tensor_act_model_lay
+{'loss': '8.105', 'grad_norm': '5.018e+05', 'learning_rate': '9e-05', 'epoch': '0.2117', 'train/total_time_seconds': '161', 'train/time_per_step_avg': '0.2391', 'train/epoch_time_elapsed': '196.3', 'train/estimated_remaining_minutes': '0.3077'}
+{'loss': '8.103', 'grad_norm': '4.321e+05', 'learning_rate': '9e-05', 'epoch': '0.213', 'train/total_time_seconds': '167.5', 'train/time_per_step_avg': '0.2533', 'train/epoch_time_elapsed': '204.2', 'train/estimated_remaining_minutes': '0.3003', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.001007', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '3.886e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.332', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6816', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5445', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.0957', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.781', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.594', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.38', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5545', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.03149', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.914', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.969', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.5', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.969', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.47', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.421e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '26.88', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '16.25', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '99', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9393', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '2.984', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '99', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '96.02', 'train/tensor_act_model_layers_0_mlp/norm': '8169', 'train/tensor_act_model_layers_0_mlp/mean': '3.984', 'train/tensor_act_model_layers_0_mlp/std': '0.03857', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '2.531', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '1.469', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8171', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '3.984', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09961', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.344', 'train/tensor_act_model_layers_
+{'loss': '8.102', 'grad_norm': '3.359e+05', 'learning_rate': '9e-05', 'epoch': '0.2144', 'train/total_time_seconds': '174.6', 'train/time_per_step_avg': '0.2783', 'train/epoch_time_elapsed': '212.4', 'train/estimated_remaining_minutes': '0.2928'}
+{'loss': '8.105', 'grad_norm': '3.011e+05', 'learning_rate': '9e-05', 'epoch': '0.2157', 'train/total_time_seconds': '181.4', 'train/time_per_step_avg': '0.3008', 'train/epoch_time_elapsed': '220.2', 'train/estimated_remaining_minutes': '0.2834'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 13.47it/s]
+ 94%|█████████▍| 3300/3500 [04:22<01:23,  2.41it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.106', 'grad_norm': '2.345e+05', 'learning_rate': '9e-05', 'epoch': '0.2171', 'train/total_time_seconds': '189.4', 'train/time_per_step_avg': '0.3297', 'train/epoch_time_elapsed': '229.5', 'train/estimated_remaining_minutes': '0.2744', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186.1', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009766', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.4', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '4.673e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.332', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.332', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.293', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5441', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.09473', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.688', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.41', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5544', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.03174', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.914', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.875', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.781', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.875', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.66', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.471e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '27.38', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '15.81', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '100.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9718', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '3.797', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '100.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '96.7', 'train/tensor_act_model_layers_0_mlp/norm': '8179', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.02234', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '2.953', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '1.047', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8181', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09424', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.281', 'train/tensor_act_model_layers_0
+{'loss': '8.104', 'grad_norm': '2.202e+05', 'learning_rate': '9e-05', 'epoch': '0.2184', 'train/total_time_seconds': '196.2', 'train/time_per_step_avg': '0.3518', 'train/epoch_time_elapsed': '237.4', 'train/estimated_remaining_minutes': '0.2624'}
+{'loss': '8.105', 'grad_norm': '1.956e+05', 'learning_rate': '9e-05', 'epoch': '0.2198', 'train/total_time_seconds': '204.4', 'train/time_per_step_avg': '0.369', 'train/epoch_time_elapsed': '246.7', 'train/estimated_remaining_minutes': '0.2508', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186.1', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009232', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.4', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '4.798e-06', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3457', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3418', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3457', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6875', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5425', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.09424', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.867', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.781', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.75', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.781', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.53', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5528', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.03052', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.906', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '8.188', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.562', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '8.188', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.75', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.477e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '27.5', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '15.56', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '106', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9863', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '4.562', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '106', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '101.4', 'train/tensor_act_model_layers_0_mlp/norm': '8183', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.01434', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.266', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.7344', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8185', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09277', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layers_0_
+{'loss': '8.105', 'grad_norm': '1.761e+05', 'learning_rate': '9e-05', 'epoch': '0.2211', 'train/total_time_seconds': '211.3', 'train/time_per_step_avg': '0.3675', 'train/epoch_time_elapsed': '254.8', 'train/estimated_remaining_minutes': '0.2363'}
+{'loss': '8.105', 'grad_norm': '1.618e+05', 'learning_rate': '9e-05', 'epoch': '0.2224', 'train/total_time_seconds': '218.1', 'train/time_per_step_avg': '0.3667', 'train/epoch_time_elapsed': '262.5', 'train/estimated_remaining_minutes': '0.2203'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 17.62it/s]
+ 97%|█████████▋| 3400/3500 [05:04<00:41,  2.43it/s][transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.104', 'grad_norm': '1.413e+05', 'learning_rate': '9e-05', 'epoch': '0.2238', 'train/total_time_seconds': '226.2', 'train/time_per_step_avg': '0.3683', 'train/epoch_time_elapsed': '271.9', 'train/estimated_remaining_minutes': '0.2044', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009918', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '6.39e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.332', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6816', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5434', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.08545', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.719', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.438', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.16', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5538', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.02991', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.914', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '8.25', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.25', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '8.25', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.5', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.497e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '27.75', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '15.31', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '94', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9914', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '5.25', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '94', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '88.75', 'train/tensor_act_model_layers_0_mlp/norm': '8185', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.01178', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.453', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.5469', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8187', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.09229', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.344', 'train/tensor_act_model_layers_0_residu
+{'loss': '8.104', 'grad_norm': '1.341e+05', 'learning_rate': '9e-05', 'epoch': '0.2251', 'train/total_time_seconds': '232.9', 'train/time_per_step_avg': '0.3666', 'train/epoch_time_elapsed': '279.7', 'train/estimated_remaining_minutes': '0.1859'}
+{'loss': '8.104', 'grad_norm': '1.224e+05', 'learning_rate': '9e-05', 'epoch': '0.2265', 'train/total_time_seconds': '240.5', 'train/time_per_step_avg': '0.3606', 'train/epoch_time_elapsed': '288.8', 'train/estimated_remaining_minutes': '0.167', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.7', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009232', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '-1.317e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3203', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3203', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.2988', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6191', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5433', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.08789', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.875', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.5', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.688', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.5', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.19', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5535', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.02966', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.914', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.906', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.594', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.906', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.5', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.506e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '28', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '15.06', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '95.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9951', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '5.781', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '95.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '89.72', 'train/tensor_act_model_layers_0_mlp/norm': '8187', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.00946', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.578', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.4219', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8189', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.0918', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layers_0_residu
+{'loss': '8.106', 'grad_norm': '1.198e+05', 'learning_rate': '9e-05', 'epoch': '0.2278', 'train/total_time_seconds': '247.3', 'train/time_per_step_avg': '0.3595', 'train/epoch_time_elapsed': '296.7', 'train/estimated_remaining_minutes': '0.1463'}
+{'loss': '8.106', 'grad_norm': '1.157e+05', 'learning_rate': '9e-05', 'epoch': '0.2292', 'train/total_time_seconds': '254.2', 'train/time_per_step_avg': '0.3616', 'train/epoch_time_elapsed': '304.7', 'train/estimated_remaining_minutes': '0.1246'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.19it/s]
+100%|██████████| 3500/3500 [06:07<00:00,  2.41i[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+{'loss': '8.106', 'grad_norm': '1.034e+05', 'learning_rate': '9e-05', 'epoch': '0.2305', 'train/total_time_seconds': '261.8', 'train/time_per_step_avg': '0.3564', 'train/epoch_time_elapsed': '313.6', 'train/estimated_remaining_minutes': '0.1021', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '185.9', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.0009308', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.1', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '-1.061e-05', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.3203', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3496', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6699', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5415', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.08252', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.867', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.688', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.625', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.31', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5517', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.02844', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.906', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.781', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.312', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.781', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.09', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.468e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '27.88', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '14.88', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '100', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9964', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '6.188', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '100', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '93.81', 'train/tensor_act_model_layers_0_mlp/norm': '8187', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.008362', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.656', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.3438', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8189', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.0918', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layer
+{'loss': '8.105', 'grad_norm': '1.014e+05', 'learning_rate': '9e-05', 'epoch': '0.2319', 'train/total_time_seconds': '268.7', 'train/time_per_step_avg': '0.3582', 'train/epoch_time_elapsed': '321.5', 'train/estimated_remaining_minutes': '0.07811'}
+{'loss': '8.108', 'grad_norm': '9.677e+04', 'learning_rate': '9e-05', 'epoch': '0.2332', 'train/total_time_seconds': '276.8', 'train/time_per_step_avg': '0.3638', 'train/epoch_time_elapsed': '330.8', 'train/estimated_remaining_minutes': '0.05334', 'train/tensor_act_model_layers_0_residual_pre_attn/norm': '186', 'train/tensor_act_model_layers_0_residual_pre_attn/mean': '0.00106', 'train/tensor_act_model_layers_0_residual_pre_attn/std': '0.09082', 'train/tensor_act_model_layers_0_residual_pre_attn/max_abs': '0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_pre_attn/min': '-0.3008', 'train/tensor_act_model_layers_0_residual_pre_attn/max': '0.2852', 'train/tensor_act_model_layers_0_residual_pre_attn/range': '0.5859', 'train/tensor_act_model_layers_0_residual_post_attn/norm': '187.3', 'train/tensor_act_model_layers_0_residual_post_attn/mean': '0.0001574', 'train/tensor_act_model_layers_0_residual_post_attn/std': '0.09131', 'train/tensor_act_model_layers_0_residual_post_attn/max_abs': '0.332', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_residual_post_attn/min': '-0.332', 'train/tensor_act_model_layers_0_residual_post_attn/max': '0.3047', 'train/tensor_act_model_layers_0_residual_post_attn/range': '0.6367', 'train/tensor_act_model_layers_0_mlp_gate_proj/norm': '5389', 'train/tensor_act_model_layers_0_mlp_gate_proj/mean': '0.08203', 'train/tensor_act_model_layers_0_mlp_gate_proj/std': '1.859', 'train/tensor_act_model_layers_0_mlp_gate_proj/max_abs': '7.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_gate_proj/min': '-6.688', 'train/tensor_act_model_layers_0_mlp_gate_proj/max': '7.469', 'train/tensor_act_model_layers_0_mlp_gate_proj/range': '14.16', 'train/tensor_act_model_layers_0_mlp_up_proj/norm': '5488', 'train/tensor_act_model_layers_0_mlp_up_proj/mean': '0.02856', 'train/tensor_act_model_layers_0_mlp_up_proj/std': '1.898', 'train/tensor_act_model_layers_0_mlp_up_proj/max_abs': '7.906', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp_up_proj/min': '-7.531', 'train/tensor_act_model_layers_0_mlp_up_proj/max': '7.906', 'train/tensor_act_model_layers_0_mlp_up_proj/range': '15.44', 'train/tensor_act_model_layers_0_mlp_down_proj/norm': '6.424e+04', 'train/tensor_act_model_layers_0_mlp_down_proj/mean': '27.75', 'train/tensor_act_model_layers_0_mlp_down_proj/std': '14.69', 'train/tensor_act_model_layers_0_mlp_down_proj/max_abs': '99.5', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp_down_proj/frac_near_user_limit': '0.9977', 'train/tensor_act_model_layers_0_mlp_down_proj/min': '6.562', 'train/tensor_act_model_layers_0_mlp_down_proj/max': '99.5', 'train/tensor_act_model_layers_0_mlp_down_proj/range': '92.94', 'train/tensor_act_model_layers_0_mlp/norm': '8188', 'train/tensor_act_model_layers_0_mlp/mean': '4', 'train/tensor_act_model_layers_0_mlp/std': '0.007172', 'train/tensor_act_model_layers_0_mlp/max_abs': '4', 'train/tensor_act_model_layers_0_mlp/frac_near_dtype_limit': '0', 'train/tensor_act_model_layers_0_mlp/frac_near_user_limit': '0', 'train/tensor_act_model_layers_0_mlp/min': '3.703', 'train/tensor_act_model_layers_0_mlp/max': '4', 'train/tensor_act_model_layers_0_mlp/range': '0.2969', 'train/tensor_act_model_layers_0_residual_post_mlp/norm': '8190', 'train/tensor_act_model_layers_0_residual_post_mlp/mean': '4', 'train/tensor_act_model_layers_0_residual_post_mlp/std': '0.0918', 'train/tensor_act_model_layers_0_residual_post_mlp/max_abs': '4.312', 'train/tensor_act_model_layers_0_
+{'loss': '8.108', 'grad_norm': '9.779e+04', 'learning_rate': '9e-05', 'epoch': '0.2346', 'train/total_time_seconds': '283.8', 'train/time_per_step_avg': '0.365', 'train/epoch_time_elapsed': '338.8', 'train/estimated_remaining_minutes': '0.02718'}
+{'loss': '8.109', 'grad_norm': '8.653e+04', 'learning_rate': '9e-05', 'epoch': '0.2359', 'train/total_time_seconds': '290.7', 'train/time_per_step_avg': '0.3643', 'train/epoch_time_elapsed': '346.6', 'train/estimated_remaining_minutes': '0'}
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+{'eval_loss': '8.104', 'eval_runtime': '20.59', 'eval_samples_per_second': '462.8', 'eval_steps_per_second': '0.486', 'epoch': '0.2359', 'train/total_time_seconds': '290.7', 'train/time_per_step_avg': '0.3643', 'train/epoch_time_elapsed': '367.2', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00,  2.60it/s]
+100%|██████████| 3500/3500 [06:07<00:00,  9.52it/s]0:00,  2.60it/s]
+{'train_runtime': '368.5', 'train_samples_per_second': '607.8', 'train_steps_per_second': '9.497', 'train_loss': '2.204', 'epoch': '0.2359', 'train/total_time_seconds': '290.7', 'train/time_per_step_avg': '0.3643', 'train/epoch_time_elapsed': '367.7', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 10/10 [00:18<00:00,  1.84s/it]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 15.55it/s]
+>>> FINISHED glu-silu-waleed10-100L successfully