diff --git "a/debug.log" "b/debug.log"
--- "a/debug.log"
+++ "b/debug.log"
@@ -1,12 +1,12 @@
-[2025-12-28 11:04:35,744] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:42410] baseline 0.000GB ()
-[2025-12-28 11:04:35,746] [INFO] [axolotl.cli.config.load_cfg:256] [PID:42410] config:
+[2025-12-29 02:49:24,896] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:3751] baseline 0.000GB ()
+[2025-12-29 02:49:24,896] [INFO] [axolotl.cli.config.load_cfg:256] [PID:3751] config:
 {
   "activation_offloading": false,
   "adapter": "lora",
   "axolotl_config_path": "tuner.yaml",
   "base_model": "codellama/CodeLlama-7b-hf",
   "base_model_config": "codellama/CodeLlama-7b-hf",
-  "batch_size": 8,
+  "batch_size": 5,
   "bf16": true,
   "capabilities": {
     "bf16": true,
@@ -17,10 +17,9 @@
   },
   "chat_template": "llama3",
   "context_parallel_size": 1,
-  "dataloader_num_workers": 1,
+  "dataloader_num_workers": 2,
   "dataloader_pin_memory": true,
-  "dataloader_prefetch_factor": 256,
-  "dataset_num_proc": 384,
+  "dataset_num_proc": 96,
   "datasets": [
     {
       "chat_template": "tokenizer_default",
@@ -36,42 +35,13 @@
     }
   ],
   "ddp": false,
-  "deepspeed": {
-    "bf16": {
-      "enabled": true
-    },
-    "fp16": {
-      "enabled": false
-    },
-    "gradient_accumulation_steps": "auto",
-    "gradient_clipping": 1.0,
-    "steps_per_print": 2000,
-    "train_micro_batch_size_per_gpu": "auto",
-    "zero_optimization": {
-      "contiguous_gradients": true,
-      "gather_16bit_weights_on_model_save": true,
-      "offload_optimizer": {
-        "device": "cpu",
-        "pin_memory": true
-      },
-      "offload_param": {
-        "device": "cpu",
-        "pin_memory": true
-      },
-      "overlap_comm": true,
-      "reduce_bucket_size": "auto",
-      "stage": 3,
-      "stage3_param_persistence_threshold": "auto",
-      "stage3_prefetch_bucket_size": "auto"
-    }
-  },
   "device": "cuda:0",
   "dion_rank_fraction": 1.0,
   "dion_rank_multiple_of": 1,
   "env_capabilities": {
     "torch_version": "2.8.0"
   },
-  "eval_batch_size": 4,
+  "eval_batch_size": 5,
   "eval_causal_lm_metrics": [
     "sacrebleu",
     "comet",
@@ -80,18 +50,16 @@
   ],
   "eval_max_new_tokens": 128,
   "eval_sample_packing": true,
-  "eval_steps": 100,
+  "eval_steps": 1000,
   "eval_table_size": 0,
   "experimental_skip_move_to_device": true,
   "fp16": false,
-  "gradient_accumulation_steps": 2,
+  "gradient_accumulation_steps": 1,
   "gradient_checkpointing": true,
   "gradient_checkpointing_kwargs": {
     "use_reentrant": true
   },
   "group_by_length": true,
-  "hub_model_id": "darwinkernelpanic/luau-codellama-7b-reasoning",
-  "hub_strategy": "every_save",
   "include_tkps": true,
   "is_falcon_derived_model": false,
   "is_llama_derived_model": true,
@@ -102,26 +70,26 @@
   "load_in_4bit": false,
   "load_in_8bit": false,
   "local_rank": 0,
-  "logging_steps": 1,
+  "logging_steps": 25,
   "lora_alpha": 32,
   "lora_dropout": 0.05,
   "lora_r": 16,
   "lora_target_modules": [
     "q_proj",
-    "v_proj",
     "k_proj",
+    "v_proj",
     "o_proj"
   ],
   "loraplus_lr_embedding": 1e-06,
   "lr_scheduler": "cosine",
   "mean_resizing_embeddings": false,
-  "micro_batch_size": 4,
+  "micro_batch_size": 5,
   "model_config_type": "llama",
   "num_epochs": 3.0,
   "optimizer": "adamw_torch",
   "otel_metrics_host": "localhost",
   "otel_metrics_port": 8000,
-  "output_dir": "./outputs/luau-codellama-h200",
+  "output_dir": "./outputs/luau-codellama-h200-fast",
   "pad_to_sequence_len": true,
   "pretrain_multipack_attn": true,
   "profiler_steps_start": 0,
@@ -135,7 +103,7 @@
   "sample_packing_group_size": 100000,
   "save_only_model": false,
   "save_safetensors": true,
-  "save_steps": 200,
+  "save_steps": 1000,
   "save_strategy": "steps",
   "save_total_limit": 3,
   "seed": 42,
@@ -175,19 +143,19 @@
     "host": "0.0.0.0",
     "port": 8000
   },
-  "warmup_steps": 10,
+  "warmup_steps": 100,
   "weight_decay": 0.0,
   "world_size": 1
 }
-[2025-12-28 11:04:36,377] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:42410] EOS: 2 / </s>
-[2025-12-28 11:04:36,378] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:42410] BOS: 1 / <s>
-[2025-12-28 11:04:36,378] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:282] [PID:42410] PAD: 2 / </s>
-[2025-12-28 11:04:36,378] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:283] [PID:42410] UNK: 0 / <unk>
-[2025-12-28 11:04:36,378] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:42410] Unable to find prepared dataset in last_run_prepared/b7c17715ff7f64badeb455c51ab5d648
-[2025-12-28 11:04:36,378] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:42410] Loading raw datasets...
-[2025-12-28 11:04:36,378] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:42410] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
-[2025-12-28 11:04:38,127] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:42410] Loading dataset: darwinkernelpanic/luau-reasoning-normalized with base_type: chat_template and prompt_style: None
-[2025-12-28 11:04:38,130] [INFO] [axolotl.prompt_strategies.chat_template.__call__:996] [PID:42410] Using chat template:
+[2025-12-29 02:49:25,389] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:3751] EOS: 2 / </s>
+[2025-12-29 02:49:25,389] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:3751] BOS: 1 / <s>
+[2025-12-29 02:49:25,389] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:282] [PID:3751] PAD: 2 / </s>
+[2025-12-29 02:49:25,389] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:283] [PID:3751] UNK: 0 / <unk>
+[2025-12-29 02:49:25,390] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:3751] Unable to find prepared dataset in last_run_prepared/b7c17715ff7f64badeb455c51ab5d648
+[2025-12-29 02:49:25,390] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:3751] Loading raw datasets...
+[2025-12-29 02:49:25,390] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:3751] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
+[2025-12-29 02:49:26,885] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:3751] Loading dataset: darwinkernelpanic/luau-reasoning-normalized with base_type: chat_template and prompt_style: None
+[2025-12-29 02:49:26,887] [INFO] [axolotl.prompt_strategies.chat_template.__call__:996] [PID:3751] Using chat template:
 ---
 {% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
 
@@ -196,1771 +164,561 @@
 ' }}{% endif %}
 
 ---
-[2025-12-28 11:04:38,137] [WARNING] [axolotl.prompt_strategies.chat_template._validate_eot_and_eos_tokens:337] [PID:42410] EOS token '</s>' not found in chat_template. Please check if your template/EOS token is correct.
-[2025-12-28 11:04:38,508] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:218] [PID:42410] min_input_len: 636
-[2025-12-28 11:04:38,508] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:220] [PID:42410] max_input_len: 12839
-[2025-12-28 11:04:41,234] [WARNING] [axolotl.utils.data.utils.handle_long_seq_in_dataset:260] [PID:42410] Dropped 755 samples from dataset
-
Saving the dataset (0/56 shards):   0%|                                                                                   | 0/14586 [00:00<?, ? examples/s]
Saving the dataset (0/56 shards):   2%|█▎                                                                      | 261/14586 [00:01<00:55, 257.60 examples/s]
Saving the dataset (1/56 shards):   2%|█▎                                                                      | 261/14586 [00:01<00:55, 257.60 examples/s]
Saving the dataset (2/56 shards):   9%|██████▎                                                                | 1305/14586 [00:01<00:51, 257.60 examples/s]
Saving the dataset (3/56 shards):   9%|██████▎                                                                | 1305/14586 [00:01<00:51, 257.60 examples/s]
Saving the dataset (4/56 shards):   9%|██████▎                                                                | 1305/14586 [00:01<00:51, 257.60 examples/s]
Saving the dataset (5/56 shards):   9%|██████▎                                                                | 1305/14586 [00:01<00:51, 257.60 examples/s]
Saving the dataset (6/56 shards):  11%|███████▌                                                               | 1566/14586 [00:01<00:50, 257.60 examples/s]
Saving the dataset (7/56 shards):  14%|██████████▏                                                            | 2088/14586 [00:01<00:48, 257.60 examples/s]
Saving the dataset (8/56 shards):  18%|████████████▋                                                          | 2610/14586 [00:01<00:46, 257.60 examples/s]
Saving the dataset (9/56 shards):  18%|████████████▋                                                          | 2610/14586 [00:01<00:46, 257.60 examples/s]
Saving the dataset (10/56 shards):  18%|████████████▌                                                         | 2610/14586 [00:01<00:46, 257.60 examples/s]
Saving the dataset (11/56 shards):  21%|███████████████                                                       | 3132/14586 [00:01<00:44, 257.60 examples/s]
Saving the dataset (12/56 shards):  21%|███████████████                                                       | 3132/14586 [00:01<00:44, 257.60 examples/s]
Saving the dataset (13/56 shards):  27%|██████████████████▊                                                   | 3915/14586 [00:01<00:41, 257.60 examples/s]
Saving the dataset (14/56 shards):  27%|██████████████████▊                                                   | 3915/14586 [00:01<00:41, 257.60 examples/s]
Saving the dataset (15/56 shards):  29%|████████████████████                                                  | 4176/14586 [00:01<00:40, 257.60 examples/s]
Saving the dataset (16/56 shards):  29%|████████████████████                                                  | 4176/14586 [00:01<00:40, 257.60 examples/s]
Saving the dataset (17/56 shards):  30%|█████████████████████▎                                                | 4437/14586 [00:01<00:39, 257.60 examples/s]
Saving the dataset (18/56 shards):  36%|█████████████████████████                                             | 5220/14586 [00:01<00:36, 257.60 examples/s]
Saving the dataset (19/56 shards):  36%|█████████████████████████                                             | 5220/14586 [00:01<00:36, 257.60 examples/s]
Saving the dataset (20/56 shards):  36%|█████████████████████████                                             | 5220/14586 [00:01<00:36, 257.60 examples/s]
Saving the dataset (21/56 shards):  39%|███████████████████████████▌                                          | 5742/14586 [00:01<00:34, 257.60 examples/s]
Saving the dataset (22/56 shards):  39%|███████████████████████████▌                                          | 5742/14586 [00:01<00:34, 257.60 examples/s]
Saving the dataset (23/56 shards):  45%|███████████████████████████████▎                                      | 6525/14586 [00:01<00:31, 257.60 examples/s]
Saving the dataset (24/56 shards):  45%|███████████████████████████████▎                                      | 6525/14586 [00:01<00:31, 257.60 examples/s]
Saving the dataset (25/56 shards):  45%|███████████████████████████████▎                                      | 6525/14586 [00:01<00:31, 257.60 examples/s]
Saving the dataset (26/56 shards):  50%|███████████████████████████████████                                   | 7306/14586 [00:01<00:28, 257.60 examples/s]
Saving the dataset (27/56 shards):  50%|███████████████████████████████████                                   | 7306/14586 [00:01<00:28, 257.60 examples/s]
Saving the dataset (28/56 shards):  50%|███████████████████████████████████                                   | 7306/14586 [00:01<00:28, 257.60 examples/s]
Saving the dataset (29/56 shards):  52%|████████████████████████████████████▎                                 | 7566/14586 [00:01<00:27, 257.60 examples/s]
Saving the dataset (30/56 shards):  55%|██████████████████████████████████████▊                               | 8086/14586 [00:01<00:25, 257.60 examples/s]
Saving the dataset (31/56 shards):  57%|████████████████████████████████████████                              | 8346/14586 [00:01<00:24, 257.60 examples/s]
Saving the dataset (32/56 shards):  57%|████████████████████████████████████████                              | 8346/14586 [00:01<00:24, 257.60 examples/s]
Saving the dataset (33/56 shards):  64%|█████████████████████████████████████████████                         | 9386/14586 [00:01<00:20, 257.60 examples/s]
Saving the dataset (34/56 shards):  64%|█████████████████████████████████████████████                         | 9386/14586 [00:01<00:20, 257.60 examples/s]
Saving the dataset (35/56 shards):  64%|█████████████████████████████████████████████                         | 9386/14586 [00:01<00:20, 257.60 examples/s]
Saving the dataset (36/56 shards):  66%|██████████████████████████████████████████████▎                       | 9646/14586 [00:01<00:19, 257.60 examples/s]
Saving the dataset (37/56 shards):  66%|██████████████████████████████████████████████▎                       | 9646/14586 [00:01<00:19, 257.60 examples/s]
Saving the dataset (38/56 shards):  68%|███████████████████████████████████████████████▌                      | 9906/14586 [00:01<00:18, 257.60 examples/s]
Saving the dataset (39/56 shards):  71%|█████████████████████████████████████████████████▎                   | 10426/14586 [00:01<00:16, 257.60 examples/s]
Saving the dataset (40/56 shards):  71%|█████████████████████████████████████████████████▎                   | 10426/14586 [00:01<00:16, 257.60 examples/s]
Saving the dataset (41/56 shards):  73%|██████████████████████████████████████████████████▌                  | 10686/14586 [00:01<00:15, 257.60 examples/s]
Saving the dataset (42/56 shards):  75%|███████████████████████████████████████████████████▊                 | 10946/14586 [00:01<00:14, 257.60 examples/s]
Saving the dataset (43/56 shards):  77%|█████████████████████████████████████████████████████                | 11206/14586 [00:01<00:13, 257.60 examples/s]
Saving the dataset (44/56 shards):  80%|███████████████████████████████████████████████████████▍             | 11726/14586 [00:01<00:11, 257.60 examples/s]
Saving the dataset (45/56 shards):  80%|███████████████████████████████████████████████████████▍             | 11726/14586 [00:01<00:11, 257.60 examples/s]
Saving the dataset (46/56 shards):  86%|███████████████████████████████████████████████████████████▏         | 12506/14586 [00:01<00:08, 257.60 examples/s]
Saving the dataset (47/56 shards):  89%|█████████████████████████████████████████████████████████████▌       | 13026/14586 [00:01<00:06, 257.60 examples/s]
Saving the dataset (48/56 shards):  91%|██████████████████████████████████████████████████████████████▊      | 13286/14586 [00:01<00:05, 257.60 examples/s]
Saving the dataset (49/56 shards):  91%|██████████████████████████████████████████████████████████████▊      | 13286/14586 [00:01<00:05, 257.60 examples/s]
Saving the dataset (50/56 shards):  91%|██████████████████████████████████████████████████████████████▊      | 13286/14586 [00:01<00:05, 257.60 examples/s]
Saving the dataset (51/56 shards):  91%|██████████████████████████████████████████████████████████████▊      | 13286/14586 [00:01<00:05, 257.60 examples/s]
Saving the dataset (52/56 shards):  95%|█████████████████████████████████████████████████████████████████▎   | 13806/14586 [00:01<00:03, 257.60 examples/s]
Saving the dataset (53/56 shards):  95%|█████████████████████████████████████████████████████████████████▎   | 13806/14586 [00:01<00:03, 257.60 examples/s]
Saving the dataset (54/56 shards):  96%|██████████████████████████████████████████████████████████████████▌  | 14066/14586 [00:01<00:02, 257.60 examples/s]
Saving the dataset (55/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:01<00:00, 257.60 examples/s]
Saving the dataset (56/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:01<00:00, 257.60 examples/s]
Saving the dataset (56/56 shards): 100%|███████████████████████████████████████████████████████████████████| 14586/14586 [00:01<00:00, 13067.01 examples/s]
-[2025-12-28 11:04:48,679] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:406] [PID:42410] total_num_tokens: 1_357_721
-[2025-12-28 11:04:48,684] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:424] [PID:42410] `total_supervised_tokens: 1_271_453`
-[2025-12-28 11:04:48,692] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:49,247] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:49,552] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.3049201965332031
-[2025-12-28 11:04:49,552] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:49,839] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.2874150276184082
-[2025-12-28 11:04:49,840] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:50,133] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.2929878234863281
-[2025-12-28 11:04:50,133] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:50,413] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.27963781356811523
-[2025-12-28 11:04:50,413] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
-[2025-12-28 11:04:50,413] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:483] [PID:42410] data_loader_len: 45
-[2025-12-28 11:04:50,413] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:499] [PID:42410] sample_packing_eff_est across ranks: [0.9131538664342287]
-[2025-12-28 11:04:50,413] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:511] [PID:42410] sample_packing_eff_est: None
-[2025-12-28 11:04:50,413] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:42410] total_num_steps: 135
-[2025-12-28 11:04:50,489] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:406] [PID:42410] total_num_tokens: 25_392_481
-[2025-12-28 11:04:50,608] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:424] [PID:42410] `total_supervised_tokens: 23_772_065`
-[2025-12-28 11:04:50,703] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:51,045] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:51,353] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.30963134765625
-[2025-12-28 11:04:51,355] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:51,664] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.31055235862731934
-[2025-12-28 11:04:51,666] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:51,976] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.3107116222381592
-[2025-12-28 11:04:51,977] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:42410] Using single process for pack_parallel, running sequentially.
-[2025-12-28 11:04:52,284] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.3080286979675293
-[2025-12-28 11:04:52,284] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [1667]
-[2025-12-28 11:04:52,284] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:483] [PID:42410] data_loader_len: 833
-[2025-12-28 11:04:52,284] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:499] [PID:42410] sample_packing_eff_est across ranks: [0.9284613122121649]
-[2025-12-28 11:04:52,284] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:511] [PID:42410] sample_packing_eff_est: 0.93
-[2025-12-28 11:04:52,285] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:42410] total_num_steps: 2499
-[2025-12-28 11:04:52,287] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:42410] Maximum number of steps set at 2499
-[2025-12-28 11:04:52,291] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:42410] loading tokenizer... codellama/CodeLlama-7b-hf
-[2025-12-28 11:04:52,784] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:42410] EOS: 2 / </s>
-[2025-12-28 11:04:52,785] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:42410] BOS: 1 / <s>
-[2025-12-28 11:04:52,785] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:282] [PID:42410] PAD: 2 / </s>
-[2025-12-28 11:04:52,785] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:283] [PID:42410] UNK: 0 / <unk>
-[2025-12-28 11:04:52,785] [DEBUG] [axolotl.train.setup_model_and_tokenizer:82] [PID:42410] Loading model
-[2025-12-28 11:04:52,926] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:42410] Patched Trainer.evaluation_loop with nanmean loss calculation
-[2025-12-28 11:04:52,927] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:42410] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
-[2025-12-28 11:04:52,927] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:301] [PID:42410] Applying multipack dataloader patch for sample packing...
-[2025-12-28 11:04:52,927] [INFO] [axolotl.loaders.patch_manager._patch_llama_sample_packing:430] [PID:42410] Patching llama _prepare_4d_causal_attention_mask*...
-
Loading checkpoint shards:   0%|                                                                                                     | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|██████████████████████████████████████████████▌                                              | 1/2 [00:01<00:01,  1.42s/it]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00,  1.14it/s]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00,  1.04it/s]
-
generation_config.json:   0%|                                                                                                    | 0.00/116 [00:00<?, ?B/s]
generation_config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 116/116 [00:00<00:00, 1.91MB/s]
-[2025-12-28 11:05:00,338] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:347] [PID:42410] Converting modules to torch.bfloat16
-[2025-12-28 11:05:00,339] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:42410] Memory usage after model load 0.500GB (+0.500GB allocated, +0.510GB reserved)
+[2025-12-29 02:49:26,891] [WARNING] [axolotl.prompt_strategies.chat_template._validate_eot_and_eos_tokens:337] [PID:3751] EOS token '</s>' not found in chat_template. Please check if your template/EOS token is correct.
+[2025-12-29 02:49:27,110] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:218] [PID:3751] min_input_len: 636
+[2025-12-29 02:49:27,110] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:220] [PID:3751] max_input_len: 12839
+[2025-12-29 02:49:27,405] [WARNING] [axolotl.utils.data.utils.handle_long_seq_in_dataset:260] [PID:3751] Dropped 755 samples from dataset
+
Saving the dataset (0/56 shards):   0%|                                                                                   | 0/14586 [00:00<?, ? examples/s]
Saving the dataset (0/56 shards):   2%|█▎                                                                      | 261/14586 [00:00<00:31, 459.80 examples/s]
Saving the dataset (1/56 shards):   9%|██████▎                                                                | 1305/14586 [00:00<00:28, 459.80 examples/s]
Saving the dataset (2/56 shards):  11%|███████▌                                                               | 1566/14586 [00:00<00:28, 459.80 examples/s]
Saving the dataset (3/56 shards):  18%|████████████▋                                                          | 2610/14586 [00:00<00:26, 459.80 examples/s]
Saving the dataset (4/56 shards):  21%|███████████████▏                                                       | 3132/14586 [00:00<00:24, 459.80 examples/s]
Saving the dataset (5/56 shards):  21%|███████████████▏                                                       | 3132/14586 [00:00<00:24, 459.80 examples/s]
Saving the dataset (6/56 shards):  34%|████████████████████████▏                                              | 4959/14586 [00:00<00:20, 459.80 examples/s]
Saving the dataset (7/56 shards):  34%|████████████████████████▏                                              | 4959/14586 [00:00<00:20, 459.80 examples/s]
Saving the dataset (8/56 shards):  38%|██████████████████████████▋                                            | 5481/14586 [00:00<00:19, 459.80 examples/s]
Saving the dataset (9/56 shards):  38%|██████████████████████████▋                                            | 5481/14586 [00:00<00:19, 459.80 examples/s]
Saving the dataset (10/56 shards):  43%|██████████████████████████████                                        | 6264/14586 [00:00<00:18, 459.80 examples/s]
Saving the dataset (11/56 shards):  79%|██████████████████████████████████████████████████████▏              | 11466/14586 [00:00<00:06, 459.80 examples/s]
Saving the dataset (12/56 shards):  79%|██████████████████████████████████████████████████████▏              | 11466/14586 [00:00<00:06, 459.80 examples/s]
Saving the dataset (13/56 shards):  95%|█████████████████████████████████████████████████████████████████▎   | 13806/14586 [00:00<00:01, 459.80 examples/s]
Saving the dataset (14/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (15/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (16/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (17/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (18/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (19/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (20/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (21/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (22/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (23/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (24/56 shards): 100%|████████████████████████████████████████████████████████████████��████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (25/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (26/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (27/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (28/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (29/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (30/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (31/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (32/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (33/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (34/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (35/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (36/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (37/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (38/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (39/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (40/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (41/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (42/56 shards): 100%|███████���█████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (43/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (44/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (45/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (46/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (47/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (48/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (49/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (50/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (51/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (52/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (53/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (54/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (55/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (56/56 shards): 100%|█████████████████████████████████████████████████████████████████████| 14586/14586 [00:00<00:00, 459.80 examples/s]
Saving the dataset (56/56 shards): 100%|███████████████████████████████████████████████████████████████████| 14586/14586 [00:01<00:00, 14250.17 examples/s]
+[2025-12-29 02:49:29,500] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:406] [PID:3751] total_num_tokens: 1_357_721
+[2025-12-29 02:49:29,511] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:424] [PID:3751] `total_supervised_tokens: 1_271_453`
+[2025-12-29 02:49:29,521] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:30,150] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:30,334] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.18436932563781738
+[2025-12-29 02:49:30,334] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:30,489] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.15522313117980957
+[2025-12-29 02:49:30,490] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:30,609] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.11953878402709961
+[2025-12-29 02:49:30,609] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:30,784] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.17447447776794434
+[2025-12-29 02:49:30,820] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:3751] gather_len_batches: [72]
+[2025-12-29 02:49:30,820] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:483] [PID:3751] data_loader_len: 72
+[2025-12-29 02:49:30,820] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:499] [PID:3751] sample_packing_eff_est across ranks: [0.9207634819878472]
+[2025-12-29 02:49:30,821] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:511] [PID:3751] sample_packing_eff_est: None
+[2025-12-29 02:49:30,821] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:3751] total_num_steps: 216
+[2025-12-29 02:49:30,904] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:406] [PID:3751] total_num_tokens: 25_392_481
+[2025-12-29 02:49:31,120] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:424] [PID:3751] `total_supervised_tokens: 23_772_065`
+[2025-12-29 02:49:31,256] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:31,508] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:31,654] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.14696550369262695
+[2025-12-29 02:49:31,656] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:31,804] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.15028667449951172
+[2025-12-29 02:49:31,806] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:31,945] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.14032292366027832
+[2025-12-29 02:49:31,947] [DEBUG] [axolotl.utils.samplers.multipack.pack_parallel:177] [PID:3751] Using single process for pack_parallel, running sequentially.
+[2025-12-29 02:49:32,123] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.17771100997924805
+[2025-12-29 02:49:32,123] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:3751] gather_len_batches: [1332]
+[2025-12-29 02:49:32,123] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:483] [PID:3751] data_loader_len: 1332
+[2025-12-29 02:49:32,123] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:499] [PID:3751] sample_packing_eff_est across ranks: [0.9298539345493663]
+[2025-12-29 02:49:32,123] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:511] [PID:3751] sample_packing_eff_est: 0.93
+[2025-12-29 02:49:32,123] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:522] [PID:3751] total_num_steps: 3996
+[2025-12-29 02:49:32,124] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:3751] Maximum number of steps set at 3996
+[2025-12-29 02:49:32,129] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:3751] loading tokenizer... codellama/CodeLlama-7b-hf
+[2025-12-29 02:49:32,522] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:3751] EOS: 2 / </s>
+[2025-12-29 02:49:32,522] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:3751] BOS: 1 / <s>
+[2025-12-29 02:49:32,522] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:282] [PID:3751] PAD: 2 / </s>
+[2025-12-29 02:49:32,522] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:283] [PID:3751] UNK: 0 / <unk>
+[2025-12-29 02:49:32,522] [DEBUG] [axolotl.train.setup_model_and_tokenizer:82] [PID:3751] Loading model
+[2025-12-29 02:49:32,640] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:3751] Patched Trainer.evaluation_loop with nanmean loss calculation
+[2025-12-29 02:49:32,642] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:3751] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
+[2025-12-29 02:49:32,642] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:301] [PID:3751] Applying multipack dataloader patch for sample packing...
+[2025-12-29 02:49:32,643] [INFO] [axolotl.loaders.patch_manager._patch_llama_sample_packing:430] [PID:3751] Patching llama _prepare_4d_causal_attention_mask*...
+
Loading checkpoint shards:   0%|                                                                                                     | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 105.69it/s]
+[2025-12-29 02:49:33,346] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:347] [PID:3751] Converting modules to torch.bfloat16
+[2025-12-29 02:49:33,680] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:3751] Memory usage after model load 0.000GB ()
 trainable params: 16,777,216 || all params: 6,755,323,904 || trainable%: 0.2484
-[2025-12-28 11:05:00,457] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:42410] after adapters 0.063GB (+0.063GB allocated, +0.572GB reserved)
-[2025-12-28 11:05:05,368] [INFO] [axolotl.train.save_initial_configs:413] [PID:42410] Pre-saving adapter config to ./outputs/luau-codellama-h200...
-[2025-12-28 11:05:05,368] [INFO] [axolotl.train.save_initial_configs:417] [PID:42410] Pre-saving tokenizer to ./outputs/luau-codellama-h200...
-[2025-12-28 11:05:05,369] [INFO] [axolotl.train.save_initial_configs:422] [PID:42410] Pre-saving model config to ./outputs/luau-codellama-h200...
-[2025-12-28 11:05:05,370] [INFO] [axolotl.train.execute_training:212] [PID:42410] Starting trainer...
-[2025-12-28 11:05:07,213] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.6575620174407959
-[2025-12-28 11:05:07,824] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.6103956699371338
-[2025-12-28 11:05:08,456] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.6313827037811279
-[2025-12-28 11:05:09,110] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.653618574142456
-[2025-12-28 11:05:09,110] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [1666]
-Parameter Offload - Persistent parameters statistics: param_count = 65, numel = 266240
-
  0%|                                                                                                                             | 0/2499 [00:00<?, ?it/s][2025-12-28 11:05:36,131] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 11:05:37,595] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.7198138236999512
-[2025-12-28 11:05:38,326] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.7297320365905762
-[2025-12-28 11:05:39,063] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.7372677326202393
-[2025-12-28 11:05:39,815] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.7512753009796143
-[2025-12-28 11:05:39,815] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
-
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:25,  3.46it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:42,  2.04it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<00:52,  1.65it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:06,  1.29it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:04,  1.29it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:04<01:07,  1.23it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:05<01:06,  1.23it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:07,  1.20it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:06,  1.21it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:07,  1.17it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:05,  1.19it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:06,  1.16it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:10<01:04,  1.18it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:11<01:05,  1.15it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:12<01:02,  1.18it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:13<01:03,  1.16it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:01,  1.18it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:01,  1.16it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<00:59,  1.18it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:16<00:59,  1.15it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:17<00:57,  1.18it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:18<00:59,  1.13it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:19<00:56,  1.16it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:57,  1.14it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.16it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:55,  1.14it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:22<00:53,  1.17it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:23<00:53,  1.14it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:24<00:51,  1.16it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:25<00:52,  1.13it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:50,  1.16it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:50,  1.13it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:48,  1.16it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:48,  1.14it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:29<00:46,  1.17it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:30<00:46,  1.14it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:31<00:44,  1.17it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:32<00:44,  1.14it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:42,  1.17it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:42,  1.15it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:40,  1.17it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:40,  1.15it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:36<00:39,  1.17it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:37<00:39,  1.15it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:38<00:37,  1.17it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.14it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:36,  1.17it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.15it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:34,  1.17it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:42<00:33,  1.15it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:43<00:32,  1.17it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:44<00:32,  1.15it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.17it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.14it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:29,  1.16it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.14it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:48<00:27,  1.16it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:49<00:27,  1.14it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:50<00:25,  1.16it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:51<00:25,  1.14it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:24,  1.16it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:53<00:23,  1.13it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:22,  1.15it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:21,  1.14it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:55<00:20,  1.16it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:56<00:20,  1.14it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:57<00:18,  1.17it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:58<00:19,  1.06it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:18,  1.11it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:17,  1.11it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.14it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.13it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.16it/s]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:03<00:13,  1.13it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:04<00:12,  1.16it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:05<00:11,  1.13it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.16it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.13it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.16it/s]
-
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 81/90 [01:09<00:07,  1.14it/s]
-
 91%|█████████████████████████��█████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:09<00:06,  1.16it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:10<00:06,  1.14it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:11<00:05,  1.16it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.13it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.16it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.15it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:15<00:01,  1.17it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89/90 [01:16<00:00,  1.15it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:16<00:00,  1.16it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 1.6886017322540283, 'eval_runtime': 79.9199, 'eval_samples_per_second': 9.134, 'eval_steps_per_second': 2.29, 'eval_ppl': 5.4119, 'memory/max_active (GiB)': 11.16, 'memory/max_allocated (GiB)': 5.19, 'memory/device_reserved (GiB)': 13.81, 'epoch': 0}
-
  0%|                                                                                                                             | 0/2499 [01:23<?, ?it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.16it/s]
-
                                                                                                                                                           
  0%|                                                                                                                  | 1/2499 [01:32<64:06:25, 92.39s/it]
                                                                                                                                                           
{'loss': 2.0336, 'grad_norm': 1.6855894327163696, 'learning_rate': 0.0, 'ppl': 7.6415, 'memory/max_active (GiB)': 16.07, 'memory/max_allocated (GiB)': 10.54, 'memory/device_reserved (GiB)': 18.02, 'tokens_per_second_per_gpu': 197269.0, 'total_tokens': 1298183, 'epoch': 0.0}
-
  0%|                                                                                                                  | 1/2499 [01:32<64:06:25, 92.39s/it]
  0%|                                                                                                                  | 2/2499 [01:38<28:57:02, 41.74s/it]
                                                                                                                                                           
{'loss': 1.7737, 'grad_norm': 1.1572575569152832, 'learning_rate': 2e-05, 'ppl': 5.8926, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 19.97, 'tokens_per_second_per_gpu': 4746.09, 'total_tokens': 1327999, 'epoch': 0.0}
-
  0%|                                                                                                                  | 2/2499 [01:38<28:57:02, 41.74s/it]
  0%|▏                                                                                                                 | 3/2499 [01:44<17:42:14, 25.53s/it]
                                                                                                                                                           
{'loss': 1.85, 'grad_norm': 1.594330072402954, 'learning_rate': 4e-05, 'ppl': 6.3598, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 19.99, 'tokens_per_second_per_gpu': 4622.21, 'total_tokens': 1356883, 'epoch': 0.0}
-
  0%|▏                                                                                                                 | 3/2499 [01:44<17:42:14, 25.53s/it]
  0%|▏                                                                                                                 | 4/2499 [01:51<12:25:28, 17.93s/it]
                                                                                                                                                           
{'loss': 1.6567, 'grad_norm': 1.557888150215149, 'learning_rate': 6e-05, 'ppl': 5.242, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 19.99, 'tokens_per_second_per_gpu': 4690.94, 'total_tokens': 1386261, 'epoch': 0.0}
-
  0%|▏                                                                                                                 | 4/2499 [01:51<12:25:28, 17.93s/it]
  0%|▏                                                                                                                  | 5/2499 [01:57<9:30:55, 13.74s/it]
                                                                                                                                                           
{'loss': 1.9046, 'grad_norm': 1.6567342281341553, 'learning_rate': 8e-05, 'ppl': 6.7167, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4507.97, 'total_tokens': 1414659, 'epoch': 0.01}
-
  0%|▏                                                                                                                  | 5/2499 [01:57<9:30:55, 13.74s/it]
  0%|▎                                                                                                                  | 6/2499 [02:03<7:45:45, 11.21s/it]
                                                                                                                                                           
{'loss': 1.8432, 'grad_norm': 1.6043676137924194, 'learning_rate': 0.0001, 'ppl': 6.3167, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4597.87, 'total_tokens': 1443641, 'epoch': 0.01}
-
  0%|▎                                                                                                                  | 6/2499 [02:03<7:45:45, 11.21s/it]
  0%|▎                                                                                                                  | 7/2499 [02:10<6:38:32,  9.60s/it]
                                                                                                                                                           
{'loss': 1.492, 'grad_norm': 1.2741687297821045, 'learning_rate': 0.00012, 'ppl': 4.446, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4542.53, 'total_tokens': 1472125, 'epoch': 0.01}
-
  0%|▎                                                                                                                  | 7/2499 [02:10<6:38:32,  9.60s/it]
  0%|▎                                                                                                                  | 8/2499 [02:16<5:54:25,  8.54s/it]
                                                                                                                                                           
{'loss': 1.4809, 'grad_norm': 1.3272074460983276, 'learning_rate': 0.00014, 'ppl': 4.3969, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4670.47, 'total_tokens': 1501396, 'epoch': 0.01}
-
  0%|▎                                                                                                                  | 8/2499 [02:16<5:54:25,  8.54s/it]
  0%|▍                                                                                                                  | 9/2499 [02:22<5:24:48,  7.83s/it]
                                                                                                                                                           
{'loss': 1.238, 'grad_norm': 1.0670270919799805, 'learning_rate': 0.00016, 'ppl': 3.4487, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4547.02, 'total_tokens': 1529874, 'epoch': 0.01}
-
  0%|▍                                                                                                                  | 9/2499 [02:22<5:24:48,  7.83s/it]
  0%|▍                                                                                                                 | 10/2499 [02:28<5:04:46,  7.35s/it]
                                                                                                                                                           
{'loss': 1.2017, 'grad_norm': 0.9426001906394958, 'learning_rate': 0.00018, 'ppl': 3.3258, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4686.7, 'total_tokens': 1559258, 'epoch': 0.01}
-
  0%|▍                                                                                                                 | 10/2499 [02:28<5:04:46,  7.35s/it]
  0%|▌                                                                                                                 | 11/2499 [02:35<4:51:01,  7.02s/it]
                                                                                                                                                           
{'loss': 1.1605, 'grad_norm': 0.8342238664627075, 'learning_rate': 0.0002, 'ppl': 3.1915, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4637.04, 'total_tokens': 1588337, 'epoch': 0.01}
-
  0%|▌                                                                                                                 | 11/2499 [02:35<4:51:01,  7.02s/it]
  0%|▌                                                                                                                 | 12/2499 [02:41<4:42:06,  6.81s/it]
                                                                                                                                                           
{'loss': 1.2037, 'grad_norm': 0.9213444590568542, 'learning_rate': 0.00019999992034374237, 'ppl': 3.3324, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4643.51, 'total_tokens': 1617675, 'epoch': 0.01}
-
  0%|▌                                                                                                                 | 12/2499 [02:41<4:42:06,  6.81s/it]
  1%|▌                                                                                                                 | 13/2499 [02:47<4:35:33,  6.65s/it]
                                                                                                                                                           
{'loss': 1.0463, 'grad_norm': 0.5648354887962341, 'learning_rate': 0.0001999996813750963, 'ppl': 2.8471, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4691.68, 'total_tokens': 1647182, 'epoch': 0.02}
-
  1%|▌                                                                                                                 | 13/2499 [02:47<4:35:33,  6.65s/it]
  1%|▋                                                                                                                 | 14/2499 [02:54<4:30:45,  6.54s/it]
                                                                                                                                                           
{'loss': 1.0009, 'grad_norm': 0.4093482494354248, 'learning_rate': 0.0001999992830944426, 'ppl': 2.7207, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4583.72, 'total_tokens': 1675932, 'epoch': 0.02}
-
  1%|▋                                                                                                                 | 14/2499 [02:54<4:30:45,  6.54s/it]
  1%|▋                                                                                                                 | 15/2499 [03:00<4:27:25,  6.46s/it]
                                                                                                                                                           
{'loss': 1.0439, 'grad_norm': 0.6911133527755737, 'learning_rate': 0.0001999987255024157, 'ppl': 2.8403, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4700.02, 'total_tokens': 1705435, 'epoch': 0.02}
-
  1%|▋                                                                                                                 | 15/2499 [03:00<4:27:25,  6.46s/it]
  1%|▋                                                                                                                 | 16/2499 [03:06<4:24:59,  6.40s/it]
                                                                                                                                                           
{'loss': 1.0052, 'grad_norm': 0.647537112236023, 'learning_rate': 0.0001999980085999039, 'ppl': 2.7325, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4640.77, 'total_tokens': 1734534, 'epoch': 0.02}
-
  1%|▋                                                                                                                 | 16/2499 [03:06<4:24:59,  6.40s/it]
  1%|▊                                                                                                                 | 17/2499 [03:12<4:23:03,  6.36s/it]
                                                                                                                                                           
{'loss': 0.8606, 'grad_norm': 0.24260607361793518, 'learning_rate': 0.0001999971323880494, 'ppl': 2.3646, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4574.84, 'total_tokens': 1763148, 'epoch': 0.02}
-
  1%|▊                                                                                                                 | 17/2499 [03:12<4:23:03,  6.36s/it]
  1%|▊                                                                                                                 | 18/2499 [03:19<4:21:50,  6.33s/it]
                                                                                                                                                           
{'loss': 0.9237, 'grad_norm': 0.34218189120292664, 'learning_rate': 0.00019999609686824802, 'ppl': 2.5186, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4659.05, 'total_tokens': 1792345, 'epoch': 0.02}
-
  1%|▊                                                                                                                 | 18/2499 [03:19<4:21:50,  6.33s/it]
  1%|▊                                                                                                                 | 19/2499 [03:25<4:21:38,  6.33s/it]
                                                                                                                                                           
{'loss': 0.8695, 'grad_norm': 0.6931776404380798, 'learning_rate': 0.00019999490204214958, 'ppl': 2.3857, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4737.23, 'total_tokens': 1822296, 'epoch': 0.02}
-
  1%|▊                                                                                                                 | 19/2499 [03:25<4:21:38,  6.33s/it]
  1%|▉                                                                                                                 | 20/2499 [03:31<4:21:01,  6.32s/it]
                                                                                                                                                           
{'loss': 0.8121, 'grad_norm': 0.29975464940071106, 'learning_rate': 0.00019999354791165749, 'ppl': 2.2526, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4681.06, 'total_tokens': 1851717, 'epoch': 0.02}
-
  1%|▉                                                                                                                 | 20/2499 [03:31<4:21:01,  6.32s/it]
  1%|▉                                                                                                                 | 21/2499 [03:38<4:20:42,  6.31s/it]
                                                                                                                                                           
{'loss': 0.8624, 'grad_norm': 0.25352585315704346, 'learning_rate': 0.0001999920344789291, 'ppl': 2.3688, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4727.48, 'total_tokens': 1881491, 'epoch': 0.03}
-
  1%|▉                                                                                                                 | 21/2499 [03:38<4:20:42,  6.31s/it]
  1%|█                                                                                                                 | 22/2499 [03:44<4:20:30,  6.31s/it]
                                                                                                                                                           
{'loss': 0.8812, 'grad_norm': 0.3950115144252777, 'learning_rate': 0.00019999036174637546, 'ppl': 2.4138, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4758.76, 'total_tokens': 1911483, 'epoch': 0.03}
-
  1%|█                                                                                                                 | 22/2499 [03:44<4:20:30,  6.31s/it]
  1%|█                                                                                                                 | 23/2499 [03:50<4:19:54,  6.30s/it]
                                                                                                                                                           
{'loss': 0.819, 'grad_norm': 0.24307860434055328, 'learning_rate': 0.0001999885297166615, 'ppl': 2.2682, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4582.96, 'total_tokens': 1940207, 'epoch': 0.03}
-
  1%|█                                                                                                                 | 23/2499 [03:50<4:19:54,  6.30s/it]
  1%|█                                                                                                                 | 24/2499 [03:56<4:19:14,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7642, 'grad_norm': 0.17830020189285278, 'learning_rate': 0.00019998653839270583, 'ppl': 2.1473, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4526.68, 'total_tokens': 1968501, 'epoch': 0.03}
-
  1%|█                                                                                                                 | 24/2499 [03:56<4:19:14,  6.28s/it]
  1%|█▏                                                                                                                | 25/2499 [04:03<4:19:12,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7952, 'grad_norm': 0.1788649559020996, 'learning_rate': 0.0001999843877776809, 'ppl': 2.2149, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4562.49, 'total_tokens': 1997194, 'epoch': 0.03}
-
  1%|█▏                                                                                                                | 25/2499 [04:03<4:19:12,  6.29s/it]
  1%|█▏                                                                                                                | 26/2499 [04:09<4:19:12,  6.29s/it]
                                                                                                                                                           
{'loss': 0.8073, 'grad_norm': 0.24912691116333008, 'learning_rate': 0.00019998207787501286, 'ppl': 2.2418, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4474.8, 'total_tokens': 2025344, 'epoch': 0.03}
-
  1%|█▏                                                                                                                | 26/2499 [04:09<4:19:12,  6.29s/it]
  1%|█▏                                                                                                                | 27/2499 [04:15<4:18:55,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7831, 'grad_norm': 0.21209484338760376, 'learning_rate': 0.00019997960868838174, 'ppl': 2.1882, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4661.31, 'total_tokens': 2054571, 'epoch': 0.03}
-
  1%|█▏                                                                                                                | 27/2499 [04:15<4:18:55,  6.28s/it]
  1%|█▎                                                                                                                | 28/2499 [04:21<4:18:38,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7746, 'grad_norm': 0.216914564371109, 'learning_rate': 0.0001999769802217212, 'ppl': 2.1697, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4733.21, 'total_tokens': 2084241, 'epoch': 0.03}
-
  1%|█▎                                                                                                                | 28/2499 [04:21<4:18:38,  6.28s/it]
  1%|█▎                                                                                                                | 29/2499 [04:28<4:18:15,  6.27s/it]
                                                                                                                                                           
{'loss': 0.8475, 'grad_norm': 0.207558274269104, 'learning_rate': 0.0001999741924792188, 'ppl': 2.3338, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4546.06, 'total_tokens': 2112679, 'epoch': 0.03}
-
  1%|█▎                                                                                                                | 29/2499 [04:28<4:18:15,  6.27s/it]
  1%|█▎                                                                                                                | 30/2499 [04:34<4:17:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.7692, 'grad_norm': 0.21438081562519073, 'learning_rate': 0.0001999712454653157, 'ppl': 2.158, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4561.33, 'total_tokens': 2141169, 'epoch': 0.04}
-
  1%|█▎                                                                                                                | 30/2499 [04:34<4:17:50,  6.27s/it]
  1%|█▍                                                                                                                | 31/2499 [04:40<4:17:31,  6.26s/it]
                                                                                                                                                           
{'loss': 0.7869, 'grad_norm': 0.16961662471294403, 'learning_rate': 0.00019996813918470686, 'ppl': 2.1966, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4358.55, 'total_tokens': 2168390, 'epoch': 0.04}
-
  1%|█▍                                                                                                                | 31/2499 [04:40<4:17:31,  6.26s/it]
  1%|█▍                                                                                                                | 32/2499 [04:47<4:17:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.8634, 'grad_norm': 0.18904076516628265, 'learning_rate': 0.000199964873642341, 'ppl': 2.3712, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4506.81, 'total_tokens': 2196748, 'epoch': 0.04}
-
  1%|█▍                                                                                                                | 32/2499 [04:47<4:17:50,  6.27s/it]
  1%|█▌                                                                                                                | 33/2499 [04:53<4:18:20,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7711, 'grad_norm': 0.16406087577342987, 'learning_rate': 0.0001999614488434205, 'ppl': 2.1621, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4605.35, 'total_tokens': 2225839, 'epoch': 0.04}
-
  1%|█▌                                                                                                                | 33/2499 [04:53<4:18:20,  6.29s/it]
  1%|█▌                                                                                                                | 34/2499 [04:59<4:18:37,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7774, 'grad_norm': 0.15022194385528564, 'learning_rate': 0.00019995786479340156, 'ppl': 2.1758, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4747.45, 'total_tokens': 2255812, 'epoch': 0.04}
-
  1%|█▌                                                                                                                | 34/2499 [04:59<4:18:37,  6.30s/it]
  1%|█▌                                                                                                                | 35/2499 [05:05<4:18:16,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6847, 'grad_norm': 0.11543940007686615, 'learning_rate': 0.00019995412149799395, 'ppl': 1.9832, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4510.07, 'total_tokens': 2284100, 'epoch': 0.04}
-
  1%|█▌                                                                                                                | 35/2499 [05:05<4:18:16,  6.29s/it]
  1%|█▋                                                                                                                | 36/2499 [05:12<4:17:51,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7787, 'grad_norm': 0.1670907884836197, 'learning_rate': 0.00019995021896316128, 'ppl': 2.1786, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4492.67, 'total_tokens': 2312230, 'epoch': 0.04}
-
  1%|█▋                                                                                                                | 36/2499 [05:12<4:17:51,  6.28s/it]
  1%|█▋                                                                                                                | 37/2499 [05:18<4:17:29,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7615, 'grad_norm': 0.19045475125312805, 'learning_rate': 0.00019994615719512072, 'ppl': 2.1415, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4537.58, 'total_tokens': 2340625, 'epoch': 0.04}
-
  1%|█▋                                                                                                                | 37/2499 [05:18<4:17:29,  6.28s/it]
  2%|█▋                                                                                                                | 38/2499 [05:24<4:17:12,  6.27s/it]
                                                                                                                                                           
{'loss': 0.841, 'grad_norm': 0.13640637695789337, 'learning_rate': 0.00019994193620034314, 'ppl': 2.3187, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4534.95, 'total_tokens': 2369006, 'epoch': 0.05}
-
  2%|█▋                                                                                                                | 38/2499 [05:24<4:17:12,  6.27s/it]
  2%|█▊                                                                                                                | 39/2499 [05:31<4:17:41,  6.28s/it]
                                                                                                                                                           
{'loss': 0.8279, 'grad_norm': 0.15840484201908112, 'learning_rate': 0.00019993755598555322, 'ppl': 2.2885, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4723.1, 'total_tokens': 2398832, 'epoch': 0.05}
-
  2%|█▊                                                                                                                | 39/2499 [05:31<4:17:41,  6.28s/it]
  2%|█▊                                                                                                                | 40/2499 [05:37<4:17:47,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6928, 'grad_norm': 0.13987034559249878, 'learning_rate': 0.0001999330165577291, 'ppl': 1.9993, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4510.37, 'total_tokens': 2427243, 'epoch': 0.05}
-
  2%|█▊                                                                                                                | 40/2499 [05:37<4:17:47,  6.29s/it]
  2%|█▊                                                                                                                | 41/2499 [05:43<4:17:35,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7248, 'grad_norm': 0.13921092450618744, 'learning_rate': 0.00019992831792410272, 'ppl': 2.0643, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4685.94, 'total_tokens': 2456661, 'epoch': 0.05}
-
  2%|█▊                                                                                                                | 41/2499 [05:43<4:17:35,  6.29s/it]
  2%|█▉                                                                                                                | 42/2499 [05:49<4:17:18,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7406, 'grad_norm': 0.12492494285106659, 'learning_rate': 0.0001999234600921595, 'ppl': 2.0972, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4612.0, 'total_tokens': 2485581, 'epoch': 0.05}
-
  2%|█▉                                                                                                                | 42/2499 [05:49<4:17:18,  6.28s/it]
  2%|█▉                                                                                                                | 43/2499 [05:56<4:17:02,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7535, 'grad_norm': 0.12467890232801437, 'learning_rate': 0.00019991844306963872, 'ppl': 2.1244, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4568.82, 'total_tokens': 2514221, 'epoch': 0.05}
-
  2%|█▉                                                                                                                | 43/2499 [05:56<4:17:02,  6.28s/it]
  2%|██                                                                                                                | 44/2499 [06:02<4:16:41,  6.27s/it]
                                                                                                                                                           
{'loss': 0.7356, 'grad_norm': 0.1306881159543991, 'learning_rate': 0.000199913266864533, 'ppl': 2.0867, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4386.29, 'total_tokens': 2541665, 'epoch': 0.05}
-
  2%|██                                                                                                                | 44/2499 [06:02<4:16:41,  6.27s/it]
  2%|██                                                                                                                | 45/2499 [06:08<4:16:23,  6.27s/it]
                                                                                                                                                           
{'loss': 0.7163, 'grad_norm': 0.1349906027317047, 'learning_rate': 0.0001999079314850887, 'ppl': 2.0468, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4404.7, 'total_tokens': 2569218, 'epoch': 0.05}
-
  2%|██                                                                                                                | 45/2499 [06:08<4:16:23,  6.27s/it]
  2%|██                                                                                                                | 46/2499 [06:15<4:16:55,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6931, 'grad_norm': 0.14203360676765442, 'learning_rate': 0.0001999024369398058, 'ppl': 1.9999, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4625.96, 'total_tokens': 2598443, 'epoch': 0.06}
-
  2%|██                                                                                                                | 46/2499 [06:15<4:16:55,  6.28s/it]
  2%|██▏                                                                                                               | 47/2499 [06:21<4:17:25,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7034, 'grad_norm': 0.1235819086432457, 'learning_rate': 0.00019989678323743774, 'ppl': 2.0206, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4575.29, 'total_tokens': 2627402, 'epoch': 0.06}
-
  2%|██▏                                                                                                               | 47/2499 [06:21<4:17:25,  6.30s/it]
  2%|██▏                                                                                                               | 48/2499 [06:27<4:17:16,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7176, 'grad_norm': 0.14084498584270477, 'learning_rate': 0.00019989097038699164, 'ppl': 2.0495, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4732.23, 'total_tokens': 2657177, 'epoch': 0.06}
-
  2%|██▏                                                                                                               | 48/2499 [06:27<4:17:16,  6.30s/it]
  2%|██▏                                                                                                               | 49/2499 [06:33<4:17:13,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7038, 'grad_norm': 0.12469019740819931, 'learning_rate': 0.00019988499839772804, 'ppl': 2.0214, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4607.81, 'total_tokens': 2686207, 'epoch': 0.06}
-
  2%|██▏                                                                                                               | 49/2499 [06:33<4:17:13,  6.30s/it]
  2%|██▎                                                                                                               | 50/2499 [06:40<4:16:45,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6652, 'grad_norm': 0.12172164767980576, 'learning_rate': 0.0001998788672791611, 'ppl': 1.9449, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4587.16, 'total_tokens': 2714952, 'epoch': 0.06}
-
  2%|██▎                                                                                                               | 50/2499 [06:40<4:16:45,  6.29s/it]
  2%|██▎                                                                                                               | 51/2499 [06:46<4:16:32,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7439, 'grad_norm': 0.12937241792678833, 'learning_rate': 0.00019987257704105844, 'ppl': 2.1041, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4660.7, 'total_tokens': 2744218, 'epoch': 0.06}
-
  2%|██▎                                                                                                               | 51/2499 [06:46<4:16:32,  6.29s/it]
  2%|██▎                                                                                                               | 52/2499 [06:52<4:16:41,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7108, 'grad_norm': 0.1375284045934677, 'learning_rate': 0.0001998661276934412, 'ppl': 2.0356, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4450.95, 'total_tokens': 2772286, 'epoch': 0.06}
-
  2%|██▎                                                                                                               | 52/2499 [06:52<4:16:41,  6.29s/it]
  2%|██▍                                                                                                               | 53/2499 [06:59<4:16:42,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7404, 'grad_norm': 0.12681901454925537, 'learning_rate': 0.000199859519246584, 'ppl': 2.0968, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4493.7, 'total_tokens': 2800601, 'epoch': 0.06}
-
  2%|██▍                                                                                                               | 53/2499 [06:59<4:16:42,  6.30s/it]
  2%|██▍                                                                                                               | 54/2499 [07:05<4:16:46,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7666, 'grad_norm': 0.1492014229297638, 'learning_rate': 0.00019985275171101495, 'ppl': 2.1524, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4497.77, 'total_tokens': 2828969, 'epoch': 0.06}
-
  2%|██▍                                                                                                               | 54/2499 [07:05<4:16:46,  6.30s/it]
  2%|██▌                                                                                                               | 55/2499 [07:11<4:16:50,  6.31s/it]
                                                                                                                                                           
{'loss': 0.723, 'grad_norm': 0.12260715663433075, 'learning_rate': 0.00019984582509751552, 'ppl': 2.0606, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4470.31, 'total_tokens': 2857188, 'epoch': 0.07}
-
  2%|██▌                                                                                                               | 55/2499 [07:11<4:16:50,  6.31s/it]
  2%|██▌                                                                                                               | 56/2499 [07:18<4:16:52,  6.31s/it]
                                                                                                                                                           
{'loss': 0.7153, 'grad_norm': 0.1511772871017456, 'learning_rate': 0.00019983873941712072, 'ppl': 2.0448, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4408.94, 'total_tokens': 2885021, 'epoch': 0.07}
-
  2%|██▌                                                                                                               | 56/2499 [07:18<4:16:52,  6.31s/it]
  2%|██▌                                                                                                               | 57/2499 [07:24<4:16:36,  6.31s/it]
                                                                                                                                                           
{'loss': 0.6686, 'grad_norm': 0.12503519654273987, 'learning_rate': 0.00019983149468111894, 'ppl': 1.9515, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4573.97, 'total_tokens': 2913799, 'epoch': 0.07}
-
  2%|██▌                                                                                                               | 57/2499 [07:24<4:16:36,  6.31s/it]
  2%|██▋                                                                                                               | 58/2499 [07:30<4:16:27,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7522, 'grad_norm': 0.12792782485485077, 'learning_rate': 0.0001998240909010519, 'ppl': 2.1217, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4703.97, 'total_tokens': 2943426, 'epoch': 0.07}
-
  2%|██▋                                                                                                               | 58/2499 [07:30<4:16:27,  6.30s/it]
  2%|██▋                                                                                                               | 59/2499 [07:36<4:16:22,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6882, 'grad_norm': 0.12937703728675842, 'learning_rate': 0.00019981652808871475, 'ppl': 1.9901, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4699.04, 'total_tokens': 2973043, 'epoch': 0.07}
-
  2%|██▋                                                                                                               | 59/2499 [07:36<4:16:22,  6.30s/it]
  2%|██▋                                                                                                               | 60/2499 [07:43<4:16:07,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6602, 'grad_norm': 0.12878933548927307, 'learning_rate': 0.00019980880625615604, 'ppl': 1.9352, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4500.91, 'total_tokens': 3001352, 'epoch': 0.07}
-
  2%|██▋                                                                                                               | 60/2499 [07:43<4:16:07,  6.30s/it]
  2%|██▊                                                                                                               | 61/2499 [07:49<4:15:36,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6685, 'grad_norm': 0.13316965103149414, 'learning_rate': 0.00019980092541567763, 'ppl': 1.9513, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4517.16, 'total_tokens': 3029652, 'epoch': 0.07}
-
  2%|██▊                                                                                                               | 61/2499 [07:49<4:15:36,  6.29s/it]
  2%|██▊                                                                                                               | 62/2499 [07:55<4:15:45,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7736, 'grad_norm': 0.12971599400043488, 'learning_rate': 0.0001997928855798346, 'ppl': 2.1676, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4444.53, 'total_tokens': 3057692, 'epoch': 0.07}
-
  2%|██▊                                                                                                               | 62/2499 [07:55<4:15:45,  6.30s/it]
  3%|██▊                                                                                                               | 63/2499 [08:02<4:15:43,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6215, 'grad_norm': 0.11753156036138535, 'learning_rate': 0.0001997846867614355, 'ppl': 1.8617, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4651.6, 'total_tokens': 3086990, 'epoch': 0.08}
-
  3%|██▊                                                                                                               | 63/2499 [08:02<4:15:43,  6.30s/it]
  3%|██▉                                                                                                               | 64/2499 [08:08<4:15:42,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6703, 'grad_norm': 0.14658862352371216, 'learning_rate': 0.00019977632897354202, 'ppl': 1.9548, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4746.31, 'total_tokens': 3116909, 'epoch': 0.08}
-
  3%|██▉                                                                                                               | 64/2499 [08:08<4:15:42,  6.30s/it]
  3%|██▉                                                                                                               | 65/2499 [08:14<4:15:28,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6798, 'grad_norm': 0.12969624996185303, 'learning_rate': 0.00019976781222946918, 'ppl': 1.9735, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4652.71, 'total_tokens': 3146161, 'epoch': 0.08}
-
  3%|██▉                                                                                                               | 65/2499 [08:14<4:15:28,  6.30s/it]
  3%|███                                                                                                               | 66/2499 [08:21<4:15:23,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6765, 'grad_norm': 0.20642466843128204, 'learning_rate': 0.00019975913654278525, 'ppl': 1.967, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4720.42, 'total_tokens': 3175889, 'epoch': 0.08}
-
  3%|███                                                                                                               | 66/2499 [08:21<4:15:23,  6.30s/it]
  3%|███                                                                                                               | 67/2499 [08:27<4:14:54,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6657, 'grad_norm': 0.12067057937383652, 'learning_rate': 0.0001997503019273116, 'ppl': 1.9459, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4506.21, 'total_tokens': 3204118, 'epoch': 0.08}
-
  3%|███                                                                                                               | 67/2499 [08:27<4:14:54,  6.29s/it]
  3%|███                                                                                                               | 68/2499 [08:33<4:14:44,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6175, 'grad_norm': 0.12278411537408829, 'learning_rate': 0.000199741308397123, 'ppl': 1.8543, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4567.1, 'total_tokens': 3232803, 'epoch': 0.08}
-
  3%|███                                                                                                               | 68/2499 [08:33<4:14:44,  6.29s/it]
  3%|███▏                                                                                                              | 69/2499 [08:39<4:15:04,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6619, 'grad_norm': 0.13150422275066376, 'learning_rate': 0.00019973215596654715, 'ppl': 1.9385, 'memory/max_active (GiB)': 17.82, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4495.32, 'total_tokens': 3261216, 'epoch': 0.08}
-
  3%|███▏                                                                                                              | 69/2499 [08:39<4:15:04,  6.30s/it]
  3%|███▏                                                                                                              | 70/2499 [08:46<4:14:55,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7218, 'grad_norm': 0.1392705738544464, 'learning_rate': 0.0001997228446501651, 'ppl': 2.0581, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4586.02, 'total_tokens': 3290070, 'epoch': 0.08}
-
  3%|███▏                                                                                                              | 70/2499 [08:46<4:14:55,  6.30s/it]
  3%|███▏                                                                                                              | 71/2499 [08:52<4:14:32,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7086, 'grad_norm': 0.15434479713439941, 'learning_rate': 0.00019971337446281087, 'ppl': 2.0311, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4580.13, 'total_tokens': 3318793, 'epoch': 0.09}
-
  3%|███▏                                                                                                              | 71/2499 [08:52<4:14:32,  6.29s/it]
  3%|███▎                                                                                                              | 72/2499 [08:58<4:14:10,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7222, 'grad_norm': 0.1450231820344925, 'learning_rate': 0.00019970374541957174, 'ppl': 2.059, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4561.96, 'total_tokens': 3347382, 'epoch': 0.09}
-
  3%|███▎                                                                                                              | 72/2499 [08:58<4:14:10,  6.28s/it]
  3%|███▎                                                                                                              | 73/2499 [09:05<4:14:10,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6646, 'grad_norm': 0.14817385375499725, 'learning_rate': 0.00019969395753578794, 'ppl': 1.9437, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4675.51, 'total_tokens': 3376788, 'epoch': 0.09}
-
  3%|███▎                                                                                                              | 73/2499 [09:05<4:14:10,  6.29s/it]
  3%|███▍                                                                                                              | 74/2499 [09:11<4:13:54,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6898, 'grad_norm': 0.131875678896904, 'learning_rate': 0.00019968401082705276, 'ppl': 1.9933, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4524.14, 'total_tokens': 3405160, 'epoch': 0.09}
-
  3%|███▍                                                                                                              | 74/2499 [09:11<4:13:54,  6.28s/it]
  3%|███▍                                                                                                              | 75/2499 [09:17<4:14:00,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6869, 'grad_norm': 0.1403125524520874, 'learning_rate': 0.0001996739053092126, 'ppl': 1.9875, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4577.87, 'total_tokens': 3433985, 'epoch': 0.09}
-
  3%|███▍                                                                                                              | 75/2499 [09:17<4:14:00,  6.29s/it]
  3%|███▍                                                                                                              | 76/2499 [09:23<4:14:26,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6758, 'grad_norm': 0.137966588139534, 'learning_rate': 0.00019966364099836681, 'ppl': 1.9656, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4609.11, 'total_tokens': 3463148, 'epoch': 0.09}
-
  3%|███▍                                                                                                              | 76/2499 [09:23<4:14:26,  6.30s/it]
  3%|███▌                                                                                                              | 77/2499 [09:30<4:14:24,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6669, 'grad_norm': 0.13154162466526031, 'learning_rate': 0.00019965321791086768, 'ppl': 1.9482, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4764.08, 'total_tokens': 3493170, 'epoch': 0.09}
-
  3%|███▌                                                                                                              | 77/2499 [09:30<4:14:24,  6.30s/it]
  3%|███▌                                                                                                              | 78/2499 [09:36<4:13:47,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6681, 'grad_norm': 0.1396287977695465, 'learning_rate': 0.00019964263606332051, 'ppl': 1.9505, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4335.49, 'total_tokens': 3520301, 'epoch': 0.09}
-
  3%|███▌                                                                                                              | 78/2499 [09:36<4:13:47,  6.29s/it]
  3%|███▌                                                                                                              | 79/2499 [09:42<4:13:27,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6799, 'grad_norm': 0.1356486976146698, 'learning_rate': 0.00019963189547258356, 'ppl': 1.9737, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4451.59, 'total_tokens': 3548202, 'epoch': 0.09}
-
  3%|███▌                                                                                                              | 79/2499 [09:42<4:13:27,  6.28s/it]
  3%|███▋                                                                                                              | 80/2499 [09:49<4:13:19,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6697, 'grad_norm': 0.14252781867980957, 'learning_rate': 0.0001996209961557679, 'ppl': 1.9537, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4500.44, 'total_tokens': 3576462, 'epoch': 0.1}
-
  3%|███▋                                                                                                              | 80/2499 [09:49<4:13:19,  6.28s/it]
  3%|███▋                                                                                                              | 81/2499 [09:55<4:13:09,  6.28s/it]
                                                                                                                                                           
{'loss': 0.7155, 'grad_norm': 0.14615966379642487, 'learning_rate': 0.00019960993813023745, 'ppl': 2.0452, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4544.14, 'total_tokens': 3604983, 'epoch': 0.1}
-
  3%|███▋                                                                                                              | 81/2499 [09:55<4:13:09,  6.28s/it]
  3%|███▋                                                                                                              | 82/2499 [10:01<4:13:23,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6172, 'grad_norm': 0.13786305487155914, 'learning_rate': 0.0001995987214136091, 'ppl': 1.8537, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4671.06, 'total_tokens': 3634442, 'epoch': 0.1}
-
  3%|███▋                                                                                                              | 82/2499 [10:01<4:13:23,  6.29s/it]
  3%|███▊                                                                                                              | 83/2499 [10:07<4:13:31,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6399, 'grad_norm': 0.14883151650428772, 'learning_rate': 0.00019958734602375247, 'ppl': 1.8963, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4347.61, 'total_tokens': 3661862, 'epoch': 0.1}
-
  3%|███▊                                                                                                              | 83/2499 [10:07<4:13:31,  6.30s/it]
  3%|███▊                                                                                                              | 84/2499 [10:14<4:13:26,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6619, 'grad_norm': 0.1344694346189499, 'learning_rate': 0.00019957581197878996, 'ppl': 1.9385, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4744.18, 'total_tokens': 3691718, 'epoch': 0.1}
-
  3%|███▊                                                                                                              | 84/2499 [10:14<4:13:26,  6.30s/it]
  3%|███▉                                                                                                              | 85/2499 [10:20<4:13:08,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7284, 'grad_norm': 0.12591156363487244, 'learning_rate': 0.00019956411929709678, 'ppl': 2.0718, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4638.44, 'total_tokens': 3720842, 'epoch': 0.1}
-
  3%|███▉                                                                                                              | 85/2499 [10:20<4:13:08,  6.29s/it]
  3%|███▉                                                                                                              | 86/2499 [10:26<4:12:45,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6502, 'grad_norm': 0.1308436542749405, 'learning_rate': 0.00019955226799730081, 'ppl': 1.9159, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4529.63, 'total_tokens': 3749228, 'epoch': 0.1}
-
  3%|███▉                                                                                                              | 86/2499 [10:26<4:12:45,  6.29s/it]
  3%|███▉                                                                                                              | 87/2499 [10:33<4:12:38,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6609, 'grad_norm': 0.13323400914669037, 'learning_rate': 0.00019954025809828266, 'ppl': 1.9365, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4566.95, 'total_tokens': 3777912, 'epoch': 0.1}
-
  3%|███▉                                                                                                              | 87/2499 [10:33<4:12:38,  6.28s/it]
  4%|████                                                                                                              | 88/2499 [10:39<4:12:35,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6446, 'grad_norm': 0.16586028039455414, 'learning_rate': 0.00019952808961917558, 'ppl': 1.9052, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4553.1, 'total_tokens': 3806539, 'epoch': 0.11}
-
  4%|████                                                                                                              | 88/2499 [10:39<4:12:35,  6.29s/it]
  4%|████                                                                                                              | 89/2499 [10:45<4:12:53,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6663, 'grad_norm': 0.14273381233215332, 'learning_rate': 0.0001995157625793655, 'ppl': 1.947, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4543.06, 'total_tokens': 3835239, 'epoch': 0.11}
-
  4%|████                                                                                                              | 89/2499 [10:45<4:12:53,  6.30s/it]
  4%|████                                                                                                              | 90/2499 [10:51<4:12:49,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6725, 'grad_norm': 0.15345992147922516, 'learning_rate': 0.00019950327699849098, 'ppl': 1.9591, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4596.19, 'total_tokens': 3864175, 'epoch': 0.11}
-
  4%|████                                                                                                              | 90/2499 [10:52<4:12:49,  6.30s/it]
  4%|████▏                                                                                                             | 91/2499 [10:58<4:12:38,  6.29s/it]
                                                                                                                                                           
{'loss': 0.7163, 'grad_norm': 0.16092751920223236, 'learning_rate': 0.00019949063289644302, 'ppl': 2.0468, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4597.41, 'total_tokens': 3893079, 'epoch': 0.11}
-
  4%|████▏                                                                                                             | 91/2499 [10:58<4:12:38,  6.29s/it]
  4%|████▏                                                                                                             | 92/2499 [11:04<4:12:21,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6764, 'grad_norm': 0.13062061369419098, 'learning_rate': 0.00019947783029336533, 'ppl': 1.9668, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4599.4, 'total_tokens': 3921954, 'epoch': 0.11}
-
  4%|████▏                                                                                                             | 92/2499 [11:04<4:12:21,  6.29s/it]
  4%|████▏                                                                                                             | 93/2499 [11:10<4:12:13,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6585, 'grad_norm': 0.14627501368522644, 'learning_rate': 0.00019946486920965404, 'ppl': 1.9319, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4612.44, 'total_tokens': 3950949, 'epoch': 0.11}
-
  4%|████▏                                                                                                             | 93/2499 [11:10<4:12:13,  6.29s/it]
  4%|████▎                                                                                                             | 94/2499 [11:17<4:12:09,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6825, 'grad_norm': 0.14802932739257812, 'learning_rate': 0.00019945174966595777, 'ppl': 1.9788, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4644.39, 'total_tokens': 3980160, 'epoch': 0.11}
-
  4%|████▎                                                                                                             | 94/2499 [11:17<4:12:09,  6.29s/it]
  4%|████▎                                                                                                             | 95/2499 [11:23<4:11:43,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6535, 'grad_norm': 0.151302307844162, 'learning_rate': 0.0001994384716831776, 'ppl': 1.9223, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4503.85, 'total_tokens': 4008363, 'epoch': 0.11}
-
  4%|████▎                                                                                                             | 95/2499 [11:23<4:11:43,  6.28s/it]
  4%|████▍                                                                                                             | 96/2499 [11:29<4:12:15,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6404, 'grad_norm': 0.15178830921649933, 'learning_rate': 0.000199425035282467, 'ppl': 1.8972, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4664.08, 'total_tokens': 4037899, 'epoch': 0.12}
-
  4%|████▍                                                                                                             | 96/2499 [11:29<4:12:15,  6.30s/it]
  4%|████▍                                                                                                             | 97/2499 [11:36<4:12:23,  6.30s/it]
                                                                                                                                                           
{'loss': 0.7097, 'grad_norm': 0.1457069218158722, 'learning_rate': 0.0001994114404852319, 'ppl': 2.0334, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4667.4, 'total_tokens': 4067373, 'epoch': 0.12}
-
  4%|████▍                                                                                                             | 97/2499 [11:36<4:12:23,  6.30s/it]
  4%|████▍                                                                                                             | 98/2499 [11:42<4:11:56,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6538, 'grad_norm': 0.13825637102127075, 'learning_rate': 0.00019939768731313046, 'ppl': 1.9228, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4609.94, 'total_tokens': 4096295, 'epoch': 0.12}
-
  4%|████▍                                                                                                             | 98/2499 [11:42<4:11:56,  6.30s/it]
  4%|████▌                                                                                                             | 99/2499 [11:48<4:11:49,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6082, 'grad_norm': 0.14136871695518494, 'learning_rate': 0.00019938377578807318, 'ppl': 1.8371, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4684.31, 'total_tokens': 4125771, 'epoch': 0.12}
-
  4%|████▌                                                                                                             | 99/2499 [11:48<4:11:49,  6.30s/it]
  4%|████▌                                                                                                            | 100/2499 [11:54<4:11:44,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6605, 'grad_norm': 0.1564965695142746, 'learning_rate': 0.0001993697059322229, 'ppl': 1.9358, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4555.83, 'total_tokens': 4154447, 'epoch': 0.12}
-
  4%|████▌                                                                                                            | 100/2499 [11:54<4:11:44,  6.30s/it][2025-12-28 11:17:31,070] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 11:17:32,807] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8711647987365723
-[2025-12-28 11:17:33,641] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8331155776977539
-[2025-12-28 11:17:34,487] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8463048934936523
-[2025-12-28 11:17:35,331] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8430600166320801
-[2025-12-28 11:17:35,331] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
-
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.42it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:55,  1.57it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<01:00,  1.42it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:06,  1.28it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:06,  1.26it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:10,  1.17it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:06<01:09,  1.19it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:07<01:10,  1.15it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:08,  1.18it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:09,  1.14it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:06,  1.17it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:07,  1.13it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:05,  1.16it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:05,  1.14it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:13<01:03,  1.16it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:14<01:04,  1.13it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:01,  1.17it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:01,  1.15it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<00:59,  1.17it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<01:00,  1.14it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:58,  1.17it/s]
-
 26%|██████████████████��███████████▏                                                                                       | 23/90 [00:19<00:58,  1.15it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:20<00:56,  1.17it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:56,  1.15it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.17it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:54,  1.15it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:52,  1.17it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:53,  1.15it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:25<00:51,  1.17it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:26<00:51,  1.15it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.17it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:49,  1.15it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:47,  1.17it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:47,  1.15it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:45,  1.17it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:45,  1.15it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:32<00:44,  1.17it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:32<00:44,  1.15it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:42,  1.17it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:42,  1.15it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:40,  1.17it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:40,  1.15it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:39,  1.17it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:38<00:38,  1.15it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:38<00:37,  1.17it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.16it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:35,  1.17it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.15it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:34,  1.18it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:33,  1.16it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:44<00:32,  1.17it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:44<00:32,  1.15it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.17it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.15it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:28,  1.17it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.14it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.17it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:27,  1.13it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:51<00:25,  1.17it/s]
-
 68%|████��██████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:51<00:25,  1.13it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:24,  1.16it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:53<00:23,  1.13it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:22,  1.16it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:22,  1.13it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:20,  1.16it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:57<00:20,  1.13it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:57<00:18,  1.16it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:58<00:18,  1.14it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.16it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.15it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.17it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:03<00:19,  1.14s/it]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:04<00:16,  1.03s/it]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:04<00:14,  1.01it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:05<00:13,  1.07it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:06<00:12,  1.08it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:07<00:10,  1.12it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:08<00:09,  1.12it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:09<00:08,  1.15it/s]
-
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 81/90 [01:10<00:07,  1.14it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:10<00:06,  1.16it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:11<00:06,  1.14it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:12<00:05,  1.16it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:13<00:04,  1.15it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:14<00:03,  1.17it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:15<00:02,  1.15it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:16<00:01,  1.17it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████��███████████████████████████████▋ | 89/90 [01:16<00:00,  1.16it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.6468729376792908, 'eval_runtime': 79.9715, 'eval_samples_per_second': 9.128, 'eval_steps_per_second': 2.288, 'eval_ppl': 1.9096, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.07, 'epoch': 0.12}
-
  4%|████▌                                                                                                            | 100/2499 [13:19<4:11:44,  6.30s/it]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:18<00:00,  1.14it/s]
-
                                                                                                                                                           
  4%|████▌                                                                                                           | 101/2499 [13:25<21:01:55, 31.57s/it]
                                                                                                                                                           
{'loss': 0.6184, 'grad_norm': 0.17828112840652466, 'learning_rate': 0.00019935547776799467, 'ppl': 1.856, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4614.55, 'total_tokens': 5453331, 'epoch': 0.12}
-
  4%|████▌                                                                                                           | 101/2499 [13:25<21:01:55, 31.57s/it]
  4%|████▌                                                                                                           | 102/2499 [13:31<15:58:35, 23.99s/it]
                                                                                                                                                           
{'loss': 0.6822, 'grad_norm': 0.2011706829071045, 'learning_rate': 0.00019934109131805575, 'ppl': 1.9782, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4637.52, 'total_tokens': 5482579, 'epoch': 0.12}
-
  4%|████▌                                                                                                           | 102/2499 [13:31<15:58:35, 23.99s/it]
  4%|████▌                                                                                                           | 103/2499 [13:38<12:26:29, 18.69s/it]
                                                                                                                                                           
{'loss': 0.5284, 'grad_norm': 0.13656415045261383, 'learning_rate': 0.00019932654660532548, 'ppl': 1.6962, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4598.33, 'total_tokens': 5511638, 'epoch': 0.12}
-
  4%|████▌                                                                                                           | 103/2499 [13:38<12:26:29, 18.69s/it]
  4%|████▋                                                                                                            | 104/2499 [13:44<9:57:26, 14.97s/it]
                                                                                                                                                           
{'loss': 0.6585, 'grad_norm': 0.15870781242847443, 'learning_rate': 0.0001993118436529755, 'ppl': 1.9319, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4352.34, 'total_tokens': 5538918, 'epoch': 0.12}
-
  4%|████▋                                                                                                            | 104/2499 [13:44<9:57:26, 14.97s/it]
  4%|████▋                                                                                                            | 105/2499 [13:50<8:13:25, 12.37s/it]
                                                                                                                                                           
{'loss': 0.6339, 'grad_norm': 0.14072741568088531, 'learning_rate': 0.00019929698248442938, 'ppl': 1.8849, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4732.02, 'total_tokens': 5568710, 'epoch': 0.13}
-
  4%|████▋                                                                                                            | 105/2499 [13:50<8:13:25, 12.37s/it]
  4%|████▊                                                                                                            | 106/2499 [13:56<7:00:22, 10.54s/it]
                                                                                                                                                           
{'loss': 0.6381, 'grad_norm': 0.14659491181373596, 'learning_rate': 0.00019928196312336285, 'ppl': 1.8929, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4575.0, 'total_tokens': 5597423, 'epoch': 0.13}
-
  4%|████▊                                                                                                            | 106/2499 [13:56<7:00:22, 10.54s/it]
  4%|████▊                                                                                                            | 107/2499 [14:03<6:09:10,  9.26s/it]
                                                                                                                                                           
{'loss': 0.6897, 'grad_norm': 0.1409890204668045, 'learning_rate': 0.00019926678559370364, 'ppl': 1.9931, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4498.1, 'total_tokens': 5625629, 'epoch': 0.13}
-
  4%|████▊                                                                                                            | 107/2499 [14:03<6:09:10,  9.26s/it]
  4%|████▉                                                                                                            | 108/2499 [14:09<5:33:17,  8.36s/it]
                                                                                                                                                           
{'loss': 0.5941, 'grad_norm': 0.1351788341999054, 'learning_rate': 0.00019925144991963145, 'ppl': 1.8114, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4593.3, 'total_tokens': 5654426, 'epoch': 0.13}
-
  4%|████▉                                                                                                            | 108/2499 [14:09<5:33:17,  8.36s/it]
  4%|████▉                                                                                                            | 109/2499 [14:15<5:08:46,  7.75s/it]
                                                                                                                                                           
{'loss': 0.6293, 'grad_norm': 0.1541460007429123, 'learning_rate': 0.00019923595612557793, 'ppl': 1.8763, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4634.55, 'total_tokens': 5683721, 'epoch': 0.13}
-
  4%|████▉                                                                                                            | 109/2499 [14:15<5:08:46,  7.75s/it]
  4%|████▉                                                                                                            | 110/2499 [14:22<4:51:31,  7.32s/it]
                                                                                                                                                           
{'loss': 0.6673, 'grad_norm': 0.17826059460639954, 'learning_rate': 0.0001992203042362266, 'ppl': 1.949, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4305.69, 'total_tokens': 5710908, 'epoch': 0.13}
-
  4%|████▉                                                                                                            | 110/2499 [14:22<4:51:31,  7.32s/it]
  4%|█████                                                                                                            | 111/2499 [14:28<4:39:24,  7.02s/it]
                                                                                                                                                           
{'loss': 0.7005, 'grad_norm': 0.14798669517040253, 'learning_rate': 0.00019920449427651292, 'ppl': 2.0148, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4490.72, 'total_tokens': 5739262, 'epoch': 0.13}
-
  4%|█████                                                                                                            | 111/2499 [14:28<4:39:24,  7.02s/it]
  4%|█████                                                                                                            | 112/2499 [14:34<4:30:29,  6.80s/it]
                                                                                                                                                           
{'loss': 0.701, 'grad_norm': 0.14876116812229156, 'learning_rate': 0.00019918852627162412, 'ppl': 2.0158, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4544.41, 'total_tokens': 5767800, 'epoch': 0.13}
-
  4%|█████                                                                                                            | 112/2499 [14:34<4:30:29,  6.80s/it]
  5%|█████                                                                                                            | 113/2499 [14:41<4:24:02,  6.64s/it]
                                                                                                                                                           
{'loss': 0.6515, 'grad_norm': 0.14015726745128632, 'learning_rate': 0.00019917240024699924, 'ppl': 1.9184, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4582.81, 'total_tokens': 5796516, 'epoch': 0.14}
-
  5%|█████                                                                                                            | 113/2499 [14:41<4:24:02,  6.64s/it]
  5%|█████▏                                                                                                           | 114/2499 [14:47<4:19:38,  6.53s/it]
                                                                                                                                                           
{'loss': 0.6357, 'grad_norm': 0.14569461345672607, 'learning_rate': 0.00019915611622832905, 'ppl': 1.8883, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4596.84, 'total_tokens': 5825374, 'epoch': 0.14}
-
  5%|█████▏                                                                                                           | 114/2499 [14:47<4:19:38,  6.53s/it]
  5%|█████▏                                                                                                           | 115/2499 [14:53<4:16:34,  6.46s/it]
                                                                                                                                                           
{'loss': 0.6666, 'grad_norm': 0.1522768884897232, 'learning_rate': 0.00019913967424155598, 'ppl': 1.9476, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4636.01, 'total_tokens': 5854490, 'epoch': 0.14}
-
  5%|█████▏                                                                                                           | 115/2499 [14:53<4:16:34,  6.46s/it]
  5%|█████▏                                                                                                           | 116/2499 [14:59<4:14:25,  6.41s/it]
                                                                                                                                                           
{'loss': 0.6558, 'grad_norm': 0.15072417259216309, 'learning_rate': 0.00019912307431287427, 'ppl': 1.9267, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4478.85, 'total_tokens': 5882638, 'epoch': 0.14}
-
  5%|█████▏                                                                                                           | 116/2499 [14:59<4:14:25,  6.41s/it]
  5%|█████▎                                                                                                           | 117/2499 [15:06<4:13:09,  6.38s/it]
                                                                                                                                                           
{'loss': 0.6541, 'grad_norm': 0.140936478972435, 'learning_rate': 0.0001991063164687296, 'ppl': 1.9234, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4528.6, 'total_tokens': 5911187, 'epoch': 0.14}
-
  5%|█████▎                                                                                                           | 117/2499 [15:06<4:13:09,  6.38s/it]
  5%|█████▎                                                                                                           | 118/2499 [15:12<4:12:05,  6.35s/it]
                                                                                                                                                           
{'loss': 0.6191, 'grad_norm': 0.14590787887573242, 'learning_rate': 0.00019908940073581937, 'ppl': 1.8573, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4668.98, 'total_tokens': 5940567, 'epoch': 0.14}
-
  5%|█████▎                                                                                                           | 118/2499 [15:12<4:12:05,  6.35s/it]
  5%|█████▍                                                                                                           | 119/2499 [15:18<4:11:21,  6.34s/it]
                                                                                                                                                           
{'loss': 0.6365, 'grad_norm': 0.13646982610225677, 'learning_rate': 0.0001990723271410924, 'ppl': 1.8899, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4828.06, 'total_tokens': 5970969, 'epoch': 0.14}
-
  5%|█████▍                                                                                                           | 119/2499 [15:18<4:11:21,  6.34s/it]
  5%|█████▍                                                                                                           | 120/2499 [15:25<4:10:27,  6.32s/it]
                                                                                                                                                           
{'loss': 0.5822, 'grad_norm': 0.1353752613067627, 'learning_rate': 0.00019905509571174914, 'ppl': 1.79, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4639.33, 'total_tokens': 6000051, 'epoch': 0.14}
-
  5%|█████▍                                                                                                           | 120/2499 [15:25<4:10:27,  6.32s/it]
  5%|█████▍                                                                                                           | 121/2499 [15:31<4:09:58,  6.31s/it]
                                                                                                                                                           
{'loss': 0.6289, 'grad_norm': 0.17556677758693695, 'learning_rate': 0.00019903770647524137, 'ppl': 1.8755, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4626.75, 'total_tokens': 6029115, 'epoch': 0.15}
-
  5%|█████▍                                                                                                           | 121/2499 [15:31<4:09:58,  6.31s/it]
  5%|█████▌                                                                                                           | 122/2499 [15:37<4:09:14,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6331, 'grad_norm': 0.1434057652950287, 'learning_rate': 0.0001990201594592723, 'ppl': 1.8834, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4451.39, 'total_tokens': 6056947, 'epoch': 0.15}
-
  5%|█████▌                                                                                                           | 122/2499 [15:37<4:09:14,  6.29s/it]
  5%|█████▌                                                                                                           | 123/2499 [15:43<4:09:19,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6185, 'grad_norm': 0.14586731791496277, 'learning_rate': 0.00019900245469179655, 'ppl': 1.8561, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4505.08, 'total_tokens': 6085351, 'epoch': 0.15}
-
  5%|█████▌                                                                                                           | 123/2499 [15:43<4:09:19,  6.30s/it]
  5%|█████▌                                                                                                           | 124/2499 [15:50<4:09:00,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6336, 'grad_norm': 0.15855848789215088, 'learning_rate': 0.00019898459220102002, 'ppl': 1.8844, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4453.23, 'total_tokens': 6113294, 'epoch': 0.15}
-
  5%|█████▌                                                                                                           | 124/2499 [15:50<4:09:00,  6.29s/it]
  5%|█████▋                                                                                                           | 125/2499 [15:56<4:08:35,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6083, 'grad_norm': 0.14481675624847412, 'learning_rate': 0.0001989665720153999, 'ppl': 1.8373, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4599.51, 'total_tokens': 6142097, 'epoch': 0.15}
-
  5%|█████▋                                                                                                           | 125/2499 [15:56<4:08:35,  6.28s/it]
  5%|█████▋                                                                                                           | 126/2499 [16:02<4:08:12,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6727, 'grad_norm': 0.167931467294693, 'learning_rate': 0.0001989483941636446, 'ppl': 1.9595, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4386.44, 'total_tokens': 6169540, 'epoch': 0.15}
-
  5%|█████▋                                                                                                           | 126/2499 [16:02<4:08:12,  6.28s/it]
  5%|█████▋                                                                                                           | 127/2499 [16:08<4:08:05,  6.28s/it]
                                                                                                                                                           
{'loss': 0.601, 'grad_norm': 0.155978262424469, 'learning_rate': 0.00019893005867471374, 'ppl': 1.8239, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4730.58, 'total_tokens': 6199215, 'epoch': 0.15}
-
  5%|█████▋                                                                                                           | 127/2499 [16:08<4:08:05,  6.28s/it]
  5%|█████▊                                                                                                           | 128/2499 [16:15<4:07:48,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6443, 'grad_norm': 0.1500401645898819, 'learning_rate': 0.00019891156557781797, 'ppl': 1.9047, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4510.45, 'total_tokens': 6227443, 'epoch': 0.15}
-
  5%|█████▊                                                                                                           | 128/2499 [16:15<4:07:48,  6.27s/it]
  5%|█████▊                                                                                                           | 129/2499 [16:21<4:07:33,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6555, 'grad_norm': 0.15343204140663147, 'learning_rate': 0.0001988929149024192, 'ppl': 1.9261, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4447.09, 'total_tokens': 6255262, 'epoch': 0.15}
-
  5%|█████▊                                                                                                           | 129/2499 [16:21<4:07:33,  6.27s/it]
  5%|█████▉                                                                                                           | 130/2499 [16:27<4:07:53,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6536, 'grad_norm': 0.18412944674491882, 'learning_rate': 0.00019887410667823022, 'ppl': 1.9224, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4639.29, 'total_tokens': 6284499, 'epoch': 0.16}
-
  5%|█████▉                                                                                                           | 130/2499 [16:27<4:07:53,  6.28s/it]
  5%|█████▉                                                                                                           | 131/2499 [16:34<4:07:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6874, 'grad_norm': 6.614463806152344, 'learning_rate': 0.00019885514093521495, 'ppl': 1.9885, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4495.37, 'total_tokens': 6312705, 'epoch': 0.16}
-
  5%|█████▉                                                                                                           | 131/2499 [16:34<4:07:47,  6.28s/it]
  5%|█████▉                                                                                                           | 132/2499 [16:40<4:07:36,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6402, 'grad_norm': 0.1778506189584732, 'learning_rate': 0.0001988360177035881, 'ppl': 1.8969, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4504.67, 'total_tokens': 6340952, 'epoch': 0.16}
-
  5%|█████▉                                                                                                           | 132/2499 [16:40<4:07:36,  6.28s/it]
  5%|██████                                                                                                           | 133/2499 [16:46<4:07:13,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5849, 'grad_norm': 0.15809500217437744, 'learning_rate': 0.00019881673701381547, 'ppl': 1.7948, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4387.74, 'total_tokens': 6368377, 'epoch': 0.16}
-
  5%|██████                                                                                                           | 133/2499 [16:46<4:07:13,  6.27s/it]
  5%|██████                                                                                                           | 134/2499 [16:52<4:07:04,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5755, 'grad_norm': 0.16758741438388824, 'learning_rate': 0.00019879729889661353, 'ppl': 1.778, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4713.74, 'total_tokens': 6397901, 'epoch': 0.16}
-
  5%|██████                                                                                                           | 134/2499 [16:52<4:07:04,  6.27s/it]
  5%|██████                                                                                                           | 135/2499 [16:59<4:06:57,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6093, 'grad_norm': 0.17591319978237152, 'learning_rate': 0.00019877770338294973, 'ppl': 1.8391, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4636.12, 'total_tokens': 6426945, 'epoch': 0.16}
-
  5%|██████                                                                                                           | 135/2499 [16:59<4:06:57,  6.27s/it]
  5%|██████▏                                                                                                          | 136/2499 [17:05<4:06:54,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6427, 'grad_norm': 0.18837158381938934, 'learning_rate': 0.0001987579505040421, 'ppl': 1.9016, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4433.54, 'total_tokens': 6454744, 'epoch': 0.16}
-
  5%|██████▏                                                                                                          | 136/2499 [17:05<4:06:54,  6.27s/it]
  5%|██████▏                                                                                                          | 137/2499 [17:11<4:07:08,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6579, 'grad_norm': 0.1512988954782486, 'learning_rate': 0.00019873804029135942, 'ppl': 1.9307, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4501.54, 'total_tokens': 6483079, 'epoch': 0.16}
-
  5%|██████▏                                                                                                          | 137/2499 [17:11<4:07:08,  6.28s/it]
  6%|██████▏                                                                                                          | 138/2499 [17:17<4:06:52,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6406, 'grad_norm': 0.1809886246919632, 'learning_rate': 0.00019871797277662125, 'ppl': 1.8976, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4511.87, 'total_tokens': 6511327, 'epoch': 0.17}
-
  6%|██████▏                                                                                                          | 138/2499 [17:17<4:06:52,  6.27s/it]
  6%|██████▎                                                                                                          | 139/2499 [17:24<4:07:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6779, 'grad_norm': 0.1574440598487854, 'learning_rate': 0.00019869774799179755, 'ppl': 1.9697, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4721.62, 'total_tokens': 6541034, 'epoch': 0.17}
-
  6%|██████▎                                                                                                          | 139/2499 [17:24<4:07:01,  6.28s/it]
  6%|██████▎                                                                                                          | 140/2499 [17:30<4:06:55,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6388, 'grad_norm': 0.16212943196296692, 'learning_rate': 0.00019867736596910902, 'ppl': 1.8942, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4727.92, 'total_tokens': 6570721, 'epoch': 0.17}
-
  6%|██████▎                                                                                                          | 140/2499 [17:30<4:06:55,  6.28s/it]
  6%|██████▍                                                                                                          | 141/2499 [17:36<4:06:44,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6901, 'grad_norm': 0.16586321592330933, 'learning_rate': 0.00019865682674102676, 'ppl': 1.9939, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4663.45, 'total_tokens': 6599963, 'epoch': 0.17}
-
  6%|██████▍                                                                                                          | 141/2499 [17:36<4:06:44,  6.28s/it]
  6%|██████▍                                                                                                          | 142/2499 [17:43<4:06:23,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6483, 'grad_norm': 0.1520916223526001, 'learning_rate': 0.00019863613034027224, 'ppl': 1.9123, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4568.36, 'total_tokens': 6628544, 'epoch': 0.17}
-
  6%|██████▍                                                                                                          | 142/2499 [17:43<4:06:23,  6.27s/it]
  6%|██████▍                                                                                                          | 143/2499 [17:49<4:07:00,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6739, 'grad_norm': 0.17079249024391174, 'learning_rate': 0.00019861527679981752, 'ppl': 1.9619, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4642.42, 'total_tokens': 6657935, 'epoch': 0.17}
-
  6%|██████▍                                                                                                          | 143/2499 [17:49<4:07:00,  6.29s/it]
  6%|██████▌                                                                                                          | 144/2499 [17:55<4:07:16,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6213, 'grad_norm': 0.14469042420387268, 'learning_rate': 0.00019859426615288488, 'ppl': 1.8613, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4454.93, 'total_tokens': 6686079, 'epoch': 0.17}
-
  6%|██████▌                                                                                                          | 144/2499 [17:55<4:07:16,  6.30s/it]
  6%|██████▌                                                                                                          | 145/2499 [18:02<4:06:50,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6334, 'grad_norm': 0.15830209851264954, 'learning_rate': 0.00019857309843294684, 'ppl': 1.884, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4440.19, 'total_tokens': 6713910, 'epoch': 0.17}
-
  6%|██████▌                                                                                                          | 145/2499 [18:02<4:06:50,  6.29s/it]
  6%|██████▌                                                                                                          | 146/2499 [18:08<4:06:32,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6419, 'grad_norm': 0.15467514097690582, 'learning_rate': 0.00019855177367372634, 'ppl': 1.9001, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4476.41, 'total_tokens': 6741989, 'epoch': 0.18}
-
  6%|██████▌                                                                                                          | 146/2499 [18:08<4:06:32,  6.29s/it]
  6%|██████▋                                                                                                          | 147/2499 [18:14<4:06:15,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6124, 'grad_norm': 0.14238551259040833, 'learning_rate': 0.0001985302919091963, 'ppl': 1.8449, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4580.51, 'total_tokens': 6770703, 'epoch': 0.18}
-
  6%|██████▋                                                                                                          | 147/2499 [18:14<4:06:15,  6.28s/it]
  6%|██████▋                                                                                                          | 148/2499 [18:20<4:06:06,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6293, 'grad_norm': 0.16102945804595947, 'learning_rate': 0.00019850865317357988, 'ppl': 1.8763, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4631.61, 'total_tokens': 6799769, 'epoch': 0.18}
-
  6%|██████▋                                                                                                          | 148/2499 [18:20<4:06:06,  6.28s/it]
  6%|██████▋                                                                                                          | 149/2499 [18:27<4:05:48,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6808, 'grad_norm': 0.1688845455646515, 'learning_rate': 0.00019848685750135033, 'ppl': 1.9755, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4488.35, 'total_tokens': 6827878, 'epoch': 0.18}
-
  6%|██████▋                                                                                                          | 149/2499 [18:27<4:05:48,  6.28s/it]
  6%|██████▊                                                                                                          | 150/2499 [18:33<4:05:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6459, 'grad_norm': 0.14278124272823334, 'learning_rate': 0.00019846490492723084, 'ppl': 1.9077, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4585.29, 'total_tokens': 6856742, 'epoch': 0.18}
-
  6%|██████▊                                                                                                          | 150/2499 [18:33<4:05:57,  6.28s/it]
  6%|██████▊                                                                                                          | 151/2499 [18:39<4:06:23,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6847, 'grad_norm': 0.1538703888654709, 'learning_rate': 0.0001984427954861946, 'ppl': 1.9832, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4651.06, 'total_tokens': 6886155, 'epoch': 0.18}
-
  6%|██████▊                                                                                                          | 151/2499 [18:39<4:06:23,  6.30s/it]
  6%|██████▊                                                                                                          | 152/2499 [18:46<4:06:17,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6242, 'grad_norm': 0.15251557528972626, 'learning_rate': 0.00019842052921346479, 'ppl': 1.8668, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4760.27, 'total_tokens': 6916113, 'epoch': 0.18}
-
  6%|██████▊                                                                                                          | 152/2499 [18:46<4:06:17,  6.30s/it]
  6%|██████▉                                                                                                          | 153/2499 [18:52<4:06:09,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6634, 'grad_norm': 0.15581682324409485, 'learning_rate': 0.00019839810614451434, 'ppl': 1.9414, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4622.18, 'total_tokens': 6945193, 'epoch': 0.18}
-
  6%|██████▉                                                                                                          | 153/2499 [18:52<4:06:09,  6.30s/it]
  6%|██████▉                                                                                                          | 154/2499 [18:58<4:05:45,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6208, 'grad_norm': 0.14313741028308868, 'learning_rate': 0.00019837552631506592, 'ppl': 1.8604, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4588.89, 'total_tokens': 6973954, 'epoch': 0.18}
-
  6%|██████▉                                                                                                          | 154/2499 [18:58<4:05:45,  6.29s/it]
  6%|███████                                                                                                          | 155/2499 [19:04<4:05:27,  6.28s/it]
                                                                                                                                                           
{'loss': 0.652, 'grad_norm': 0.14645761251449585, 'learning_rate': 0.00019835278976109214, 'ppl': 1.9194, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4630.67, 'total_tokens': 7002988, 'epoch': 0.19}
-
  6%|███████                                                                                                          | 155/2499 [19:04<4:05:27,  6.28s/it]
  6%|███████                                                                                                          | 156/2499 [19:11<4:05:11,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6053, 'grad_norm': 0.1450553685426712, 'learning_rate': 0.0001983298965188151, 'ppl': 1.8318, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4535.2, 'total_tokens': 7031406, 'epoch': 0.19}
-
  6%|███████                                                                                                          | 156/2499 [19:11<4:05:11,  6.28s/it]
  6%|███████                                                                                                          | 157/2499 [19:17<4:05:20,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6132, 'grad_norm': 0.14832331240177155, 'learning_rate': 0.00019830684662470663, 'ppl': 1.8463, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4542.32, 'total_tokens': 7060015, 'epoch': 0.19}
-
  6%|███████                                                                                                          | 157/2499 [19:17<4:05:20,  6.29s/it]
  6%|███████▏                                                                                                         | 158/2499 [19:23<4:05:24,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6337, 'grad_norm': 0.15093529224395752, 'learning_rate': 0.0001982836401154881, 'ppl': 1.8846, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4563.48, 'total_tokens': 7088745, 'epoch': 0.19}
-
  6%|███████▏                                                                                                         | 158/2499 [19:23<4:05:24,  6.29s/it]
  6%|███████▏                                                                                                         | 159/2499 [19:30<4:05:10,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6969, 'grad_norm': 0.16975665092468262, 'learning_rate': 0.00019826027702813038, 'ppl': 2.0075, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4595.17, 'total_tokens': 7117583, 'epoch': 0.19}
-
  6%|███████▏                                                                                                         | 159/2499 [19:30<4:05:10,  6.29s/it]
  6%|███████▏                                                                                                         | 160/2499 [19:36<4:04:55,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6083, 'grad_norm': 0.1516297310590744, 'learning_rate': 0.00019823675739985376, 'ppl': 1.8373, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4718.55, 'total_tokens': 7147178, 'epoch': 0.19}
-
  6%|███████▏                                                                                                         | 160/2499 [19:36<4:04:55,  6.28s/it]
  6%|███████▎                                                                                                         | 161/2499 [19:42<4:04:46,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6185, 'grad_norm': 0.14229127764701843, 'learning_rate': 0.00019821308126812803, 'ppl': 1.8561, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4711.98, 'total_tokens': 7176755, 'epoch': 0.19}
-
  6%|███████▎                                                                                                         | 161/2499 [19:42<4:04:46,  6.28s/it]
  6%|███████▎                                                                                                         | 162/2499 [19:48<4:04:20,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6169, 'grad_norm': 0.17252376675605774, 'learning_rate': 0.00019818924867067214, 'ppl': 1.8532, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4502.44, 'total_tokens': 7204896, 'epoch': 0.19}
-
  6%|███████▎                                                                                                         | 162/2499 [19:48<4:04:20,  6.27s/it]
  7%|███████▎                                                                                                         | 163/2499 [19:55<4:04:10,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6053, 'grad_norm': 0.15479132533073425, 'learning_rate': 0.00019816525964545448, 'ppl': 1.8318, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4656.1, 'total_tokens': 7234072, 'epoch': 0.2}
-
  7%|███████▎                                                                                                         | 163/2499 [19:55<4:04:10,  6.27s/it]
  7%|███████▍                                                                                                         | 164/2499 [20:01<4:04:32,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6358, 'grad_norm': 0.1458706557750702, 'learning_rate': 0.0001981411142306925, 'ppl': 1.8885, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4626.2, 'total_tokens': 7263258, 'epoch': 0.2}
-
  7%|███████▍                                                                                                         | 164/2499 [20:01<4:04:32,  6.28s/it]
  7%|███████▍                                                                                                         | 165/2499 [20:07<4:04:24,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5665, 'grad_norm': 0.1417934000492096, 'learning_rate': 0.0001981168124648529, 'ppl': 1.7621, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4551.14, 'total_tokens': 7291824, 'epoch': 0.2}
-
  7%|███████▍                                                                                                         | 165/2499 [20:07<4:04:24,  6.28s/it]
  7%|███████▌                                                                                                         | 166/2499 [20:13<4:04:06,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6314, 'grad_norm': 0.1490688920021057, 'learning_rate': 0.00019809235438665143, 'ppl': 1.8802, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4564.55, 'total_tokens': 7320418, 'epoch': 0.2}
-
  7%|███████▌                                                                                                         | 166/2499 [20:13<4:04:06,  6.28s/it]
  7%|███████▌                                                                                                         | 167/2499 [20:20<4:03:43,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6009, 'grad_norm': 0.1549319177865982, 'learning_rate': 0.0001980677400350529, 'ppl': 1.8238, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4506.43, 'total_tokens': 7348591, 'epoch': 0.2}
-
  7%|███████▌                                                                                                         | 167/2499 [20:20<4:03:43,  6.27s/it]
  7%|███████▌                                                                                                         | 168/2499 [20:26<4:03:35,  6.27s/it]
                                                                                                                                                           
{'loss': 0.582, 'grad_norm': 0.1679680198431015, 'learning_rate': 0.000198042969449271, 'ppl': 1.7896, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4706.82, 'total_tokens': 7378083, 'epoch': 0.2}
-
  7%|███████▌                                                                                                         | 168/2499 [20:26<4:03:35,  6.27s/it]
  7%|███████▋                                                                                                         | 169/2499 [20:32<4:03:31,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6688, 'grad_norm': 0.16335871815681458, 'learning_rate': 0.0001980180426687684, 'ppl': 1.9519, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4740.61, 'total_tokens': 7407810, 'epoch': 0.2}
-
  7%|███████▋                                                                                                         | 169/2499 [20:32<4:03:31,  6.27s/it]
  7%|███████▋                                                                                                         | 170/2499 [20:39<4:03:31,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5984, 'grad_norm': 0.15233907103538513, 'learning_rate': 0.00019799295973325657, 'ppl': 1.8192, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4701.67, 'total_tokens': 7437325, 'epoch': 0.2}
-
  7%|███████▋                                                                                                         | 170/2499 [20:39<4:03:31,  6.27s/it]
  7%|███████▋                                                                                                         | 171/2499 [20:45<4:03:37,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6533, 'grad_norm': 0.14838764071464539, 'learning_rate': 0.0001979677206826958, 'ppl': 1.9219, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4422.84, 'total_tokens': 7465136, 'epoch': 0.21}
-
  7%|███████▋                                                                                                         | 171/2499 [20:45<4:03:37,  6.28s/it]
  7%|███████▊                                                                                                         | 172/2499 [20:51<4:03:32,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5928, 'grad_norm': 0.1395515352487564, 'learning_rate': 0.000197942325557295, 'ppl': 1.809, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4613.25, 'total_tokens': 7494094, 'epoch': 0.21}
-
  7%|███████▊                                                                                                         | 172/2499 [20:51<4:03:32,  6.28s/it]
  7%|███████▊                                                                                                         | 173/2499 [20:57<4:03:08,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6164, 'grad_norm': 0.14091241359710693, 'learning_rate': 0.00019791677439751185, 'ppl': 1.8522, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4590.45, 'total_tokens': 7522794, 'epoch': 0.21}
-
  7%|███████▊                                                                                                         | 173/2499 [20:57<4:03:08,  6.27s/it]
  7%|███████▊                                                                                                         | 174/2499 [21:04<4:02:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6732, 'grad_norm': 0.16553938388824463, 'learning_rate': 0.0001978910672440525, 'ppl': 1.9605, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4550.86, 'total_tokens': 7551247, 'epoch': 0.21}
-
  7%|███████▊                                                                                                         | 174/2499 [21:04<4:02:50,  6.27s/it]
  7%|███████▉                                                                                                         | 175/2499 [21:10<4:02:44,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6298, 'grad_norm': 0.15987837314605713, 'learning_rate': 0.00019786520413787165, 'ppl': 1.8772, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4506.6, 'total_tokens': 7579481, 'epoch': 0.21}
-
  7%|███████▉                                                                                                         | 175/2499 [21:10<4:02:44,  6.27s/it]
  7%|███████▉                                                                                                         | 176/2499 [21:16<4:02:42,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6511, 'grad_norm': 0.14235079288482666, 'learning_rate': 0.00019783918512017253, 'ppl': 1.9176, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4668.32, 'total_tokens': 7608756, 'epoch': 0.21}
-
  7%|███████▉                                                                                                         | 176/2499 [21:16<4:02:42,  6.27s/it]
  7%|████████                                                                                                         | 177/2499 [21:22<4:03:16,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6042, 'grad_norm': 0.17243558168411255, 'learning_rate': 0.0001978130102324066, 'ppl': 1.8298, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4565.31, 'total_tokens': 7637623, 'epoch': 0.21}
-
  7%|████████                                                                                                         | 177/2499 [21:22<4:03:16,  6.29s/it]
  7%|████████                                                                                                         | 178/2499 [21:29<4:03:26,  6.29s/it]
                                                                                                                                                           
{'loss': 0.637, 'grad_norm': 0.16263476014137268, 'learning_rate': 0.00019778667951627382, 'ppl': 1.8908, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4526.74, 'total_tokens': 7666166, 'epoch': 0.21}
-
  7%|████████                                                                                                         | 178/2499 [21:29<4:03:26,  6.29s/it]
  7%|████████                                                                                                         | 179/2499 [21:35<4:03:04,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6186, 'grad_norm': 0.15282128751277924, 'learning_rate': 0.00019776019301372225, 'ppl': 1.8563, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4439.86, 'total_tokens': 7693990, 'epoch': 0.21}
-
  7%|████████                                                                                                         | 179/2499 [21:35<4:03:04,  6.29s/it]
  7%|████████▏                                                                                                        | 180/2499 [21:41<4:02:51,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6161, 'grad_norm': 0.14302721619606018, 'learning_rate': 0.00019773355076694826, 'ppl': 1.8517, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4753.91, 'total_tokens': 7723820, 'epoch': 0.22}
-
  7%|████████▏                                                                                                        | 180/2499 [21:41<4:02:51,  6.28s/it]
  7%|████████▏                                                                                                        | 181/2499 [21:48<4:02:32,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6521, 'grad_norm': 0.1567981094121933, 'learning_rate': 0.00019770675281839624, 'ppl': 1.9196, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4552.84, 'total_tokens': 7752331, 'epoch': 0.22}
-
  7%|████████▏                                                                                                        | 181/2499 [21:48<4:02:32,  6.28s/it]
  7%|████████▏                                                                                                        | 182/2499 [21:54<4:02:22,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6502, 'grad_norm': 0.16891400516033173, 'learning_rate': 0.00019767979921075866, 'ppl': 1.9159, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4489.31, 'total_tokens': 7780479, 'epoch': 0.22}
-
  7%|████████▏                                                                                                        | 182/2499 [21:54<4:02:22,  6.28s/it]
  7%|████████▎                                                                                                        | 183/2499 [22:00<4:02:16,  6.28s/it]
                                                                                                                                                           
{'loss': 0.62, 'grad_norm': 0.15879429876804352, 'learning_rate': 0.00019765268998697604, 'ppl': 1.8589, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4588.46, 'total_tokens': 7809267, 'epoch': 0.22}
-
  7%|████████▎                                                                                                        | 183/2499 [22:00<4:02:16,  6.28s/it]
  7%|████████▎                                                                                                        | 184/2499 [22:06<4:02:26,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6378, 'grad_norm': 0.1598796546459198, 'learning_rate': 0.00019762542519023674, 'ppl': 1.8923, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4576.5, 'total_tokens': 7838088, 'epoch': 0.22}
-
  7%|████████▎                                                                                                        | 184/2499 [22:06<4:02:26,  6.28s/it]
  7%|████████▎                                                                                                        | 185/2499 [22:13<4:02:51,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5584, 'grad_norm': 0.1714273989200592, 'learning_rate': 0.00019759800486397703, 'ppl': 1.7479, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4679.83, 'total_tokens': 7867688, 'epoch': 0.22}
-
  7%|████████▎                                                                                                        | 185/2499 [22:13<4:02:51,  6.30s/it]
  7%|████████▍                                                                                                        | 186/2499 [22:19<4:02:17,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6139, 'grad_norm': 0.16586022078990936, 'learning_rate': 0.00019757042905188088, 'ppl': 1.8476, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4281.2, 'total_tokens': 7894459, 'epoch': 0.22}
-
  7%|████████▍                                                                                                        | 186/2499 [22:19<4:02:17,  6.29s/it]
  7%|████████▍                                                                                                        | 187/2499 [22:25<4:02:07,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6282, 'grad_norm': 0.1663977950811386, 'learning_rate': 0.00019754269779788, 'ppl': 1.8742, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4602.7, 'total_tokens': 7923350, 'epoch': 0.22}
-
  7%|████████▍                                                                                                        | 187/2499 [22:25<4:02:07,  6.28s/it]
  8%|████████▌                                                                                                        | 188/2499 [22:32<4:01:49,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5851, 'grad_norm': 0.1668008416891098, 'learning_rate': 0.0001975148111461538, 'ppl': 1.7952, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4571.75, 'total_tokens': 7951987, 'epoch': 0.23}
-
  8%|████████▌                                                                                                        | 188/2499 [22:32<4:01:49,  6.28s/it]
  8%|████████▌                                                                                                        | 189/2499 [22:38<4:01:32,  6.27s/it]
                                                                                                                                                           
{'loss': 0.626, 'grad_norm': 0.18379661440849304, 'learning_rate': 0.00019748676914112915, 'ppl': 1.8701, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4556.79, 'total_tokens': 7980520, 'epoch': 0.23}
-
  8%|████████▌                                                                                                        | 189/2499 [22:38<4:01:32,  6.27s/it]
  8%|████████▌                                                                                                        | 190/2499 [22:44<4:01:34,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5925, 'grad_norm': 0.13806037604808807, 'learning_rate': 0.00019745857182748054, 'ppl': 1.8085, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4700.59, 'total_tokens': 8010056, 'epoch': 0.23}
-
  8%|████████▌                                                                                                        | 190/2499 [22:44<4:01:34,  6.28s/it]
  8%|████████▋                                                                                                        | 191/2499 [22:50<4:01:44,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6313, 'grad_norm': 0.14297842979431152, 'learning_rate': 0.00019743021925012973, 'ppl': 1.8801, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4553.45, 'total_tokens': 8038737, 'epoch': 0.23}
-
  8%|████████▋                                                                                                        | 191/2499 [22:50<4:01:44,  6.28s/it]
  8%|████████▋                                                                                                        | 192/2499 [22:57<4:02:09,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6269, 'grad_norm': 0.16967882215976715, 'learning_rate': 0.000197401711454246, 'ppl': 1.8718, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4698.64, 'total_tokens': 8068455, 'epoch': 0.23}
-
  8%|████████▋                                                                                                        | 192/2499 [22:57<4:02:09,  6.30s/it]
  8%|████████▋                                                                                                        | 193/2499 [23:03<4:01:52,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5853, 'grad_norm': 0.15979325771331787, 'learning_rate': 0.0001973730484852458, 'ppl': 1.7955, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4662.01, 'total_tokens': 8097728, 'epoch': 0.23}
-
  8%|████████▋                                                                                                        | 193/2499 [23:03<4:01:52,  6.29s/it]
  8%|████████▊                                                                                                        | 194/2499 [23:09<4:01:24,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6389, 'grad_norm': 0.1816360056400299, 'learning_rate': 0.00019734423038879283, 'ppl': 1.8944, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4387.73, 'total_tokens': 8125195, 'epoch': 0.23}
-
  8%|████████▊                                                                                                        | 194/2499 [23:09<4:01:24,  6.28s/it]
  8%|████████▊                                                                                                        | 195/2499 [23:16<4:01:06,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5965, 'grad_norm': 0.14533467590808868, 'learning_rate': 0.00019731525721079793, 'ppl': 1.8158, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4552.24, 'total_tokens': 8153711, 'epoch': 0.23}
-
  8%|████████▊                                                                                                        | 195/2499 [23:16<4:01:06,  6.28s/it]
  8%|████████▊                                                                                                        | 196/2499 [23:22<4:00:54,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6688, 'grad_norm': 0.16294941306114197, 'learning_rate': 0.000197286128997419, 'ppl': 1.9519, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4513.63, 'total_tokens': 8182000, 'epoch': 0.24}
-
  8%|████████▊                                                                                                        | 196/2499 [23:22<4:00:54,  6.28s/it]
  8%|████████▉                                                                                                        | 197/2499 [23:28<4:00:37,  6.27s/it]
                                                                                                                                                           
{'loss': 0.638, 'grad_norm': 0.15876515209674835, 'learning_rate': 0.00019725684579506095, 'ppl': 1.8927, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4516.46, 'total_tokens': 8210270, 'epoch': 0.24}
-
  8%|████████▉                                                                                                        | 197/2499 [23:28<4:00:37,  6.27s/it]
  8%|████████▉                                                                                                        | 198/2499 [23:34<4:01:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6244, 'grad_norm': 0.1551365852355957, 'learning_rate': 0.00019722740765037555, 'ppl': 1.8671, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4607.89, 'total_tokens': 8239361, 'epoch': 0.24}
-
  8%|████████▉                                                                                                        | 198/2499 [23:34<4:01:01,  6.28s/it]
  8%|████████▉                                                                                                        | 199/2499 [23:41<4:01:12,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6537, 'grad_norm': 0.15418943762779236, 'learning_rate': 0.00019719781461026146, 'ppl': 1.9226, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4639.8, 'total_tokens': 8268621, 'epoch': 0.24}
-
  8%|████████▉                                                                                                        | 199/2499 [23:41<4:01:12,  6.29s/it]
  8%|█████████                                                                                                        | 200/2499 [23:47<4:00:55,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6467, 'grad_norm': 0.15851524472236633, 'learning_rate': 0.00019716806672186412, 'ppl': 1.9092, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4664.04, 'total_tokens': 8297884, 'epoch': 0.24}
-
  8%|█████████                                                                                                        | 200/2499 [23:47<4:00:55,  6.29s/it][2025-12-28 11:29:23,624] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 11:29:25,368] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8519337177276611
-[2025-12-28 11:29:26,219] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8512239456176758
-[2025-12-28 11:29:27,107] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8861675262451172
-[2025-12-28 11:29:27,946] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8390281200408936
-[2025-12-28 11:29:27,947] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
-
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.41it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:54,  1.58it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<01:00,  1.43it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:05,  1.30it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:05,  1.27it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:08,  1.22it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:06<01:07,  1.22it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:08,  1.19it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:06,  1.20it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:07,  1.17it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:05,  1.19it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:06,  1.16it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:04,  1.18it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:04,  1.16it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:12<01:02,  1.18it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:13<01:02,  1.16it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:01,  1.18it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:01,  1.16it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<00:59,  1.18it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<00:59,  1.15it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:58,  1.17it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:18<00:58,  1.15it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:19<00:56,  1.17it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:56,  1.15it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.17it/s]
-
 30%|█████████��█████████████████████████▍                                                                                  | 27/90 [00:22<00:55,  1.14it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:53,  1.17it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:53,  1.13it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:24<00:51,  1.16it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:25<00:51,  1.14it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.16it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:50,  1.13it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:48,  1.16it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:48,  1.13it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:46,  1.17it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:46,  1.15it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:31<00:44,  1.17it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:32<00:44,  1.15it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:42,  1.17it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:42,  1.15it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:41,  1.17it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:40,  1.15it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:39,  1.16it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:38<00:39,  1.14it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:38<00:37,  1.16it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.15it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:35,  1.17it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.15it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:34,  1.17it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:34,  1.14it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:43<00:32,  1.17it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:44<00:32,  1.15it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.17it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:47<00:34,  1.01it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:28,  1.18it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.16it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.18it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:26,  1.16it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:50<00:25,  1.18it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:51<00:25,  1.15it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:23,  1.17it/s]
-
 70%|█████████████████████████████████████████████████████████████████████████████████��▌                                   | 63/90 [00:53<00:26,  1.02it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:24,  1.07it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:23,  1.09it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:21,  1.12it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:57<00:20,  1.12it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:58<00:19,  1.15it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:59<00:18,  1.14it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.16it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.15it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.17it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.14it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.17it/s]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:04<00:13,  1.14it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:05<00:11,  1.17it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:06<00:11,  1.14it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.17it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.14it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.17it/s]
-
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 81/90 [01:09<00:07,  1.14it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:10<00:06,  1.17it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:11<00:06,  1.14it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:12<00:05,  1.16it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.14it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.16it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.15it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:15<00:01,  1.17it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89/90 [01:16<00:00,  1.15it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.15it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.6098045110702515, 'eval_runtime': 79.6449, 'eval_samples_per_second': 9.166, 'eval_steps_per_second': 2.298, 'eval_ppl': 1.8401, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.19, 'epoch': 0.24}
-
  8%|█████████                                                                                                        | 200/2499 [25:11<4:00:55,  6.29s/it]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.15it/s]
-
                                                                                                                                                           [2025-12-28 11:30:51,250] [INFO] [axolotl.core.trainers.base._save:692] [PID:42410] Saving model checkpoint to ./outputs/luau-codellama-h200/checkpoint-200
-
  8%|█████████                                                                                                       | 201/2499 [25:22<20:58:02, 32.85s/it]
                                                                                                                                                           
{'loss': 0.62, 'grad_norm': 0.1783120483160019, 'learning_rate': 0.0001971381640325756, 'ppl': 1.8589, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4565.79, 'total_tokens': 9596269, 'epoch': 0.24}
-
  8%|█████████                                                                                                       | 201/2499 [25:22<20:58:02, 32.85s/it]
  8%|█████████                                                                                                       | 202/2499 [25:28<15:52:06, 24.87s/it]
                                                                                                                                                           
{'loss': 0.6069, 'grad_norm': 0.1567048877477646, 'learning_rate': 0.00019710810659003463, 'ppl': 1.8347, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4530.52, 'total_tokens': 9624606, 'epoch': 0.24}
-
  8%|█████████                                                                                                       | 202/2499 [25:28<15:52:06, 24.87s/it]
  8%|█████████                                                                                                       | 203/2499 [25:34<12:18:04, 19.29s/it]
                                                                                                                                                           
{'loss': 0.6501, 'grad_norm': 0.17752958834171295, 'learning_rate': 0.00019707789444212655, 'ppl': 1.9157, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4547.65, 'total_tokens': 9653076, 'epoch': 0.24}
-
  8%|█████████                                                                                                       | 203/2499 [25:34<12:18:04, 19.29s/it]
  8%|█████████▏                                                                                                       | 204/2499 [25:41<9:49:16, 15.41s/it]
                                                                                                                                                           
{'loss': 0.6438, 'grad_norm': 0.16355815529823303, 'learning_rate': 0.00019704752763698301, 'ppl': 1.9037, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4655.9, 'total_tokens': 9682621, 'epoch': 0.24}
-
  8%|█████████▏                                                                                                       | 204/2499 [25:41<9:49:16, 15.41s/it]
  8%|█████████▎                                                                                                       | 205/2499 [25:47<8:05:07, 12.69s/it]
                                                                                                                                                           
{'loss': 0.6053, 'grad_norm': 0.15781280398368835, 'learning_rate': 0.00019701700622298213, 'ppl': 1.8318, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4721.86, 'total_tokens': 9712580, 'epoch': 0.25}
-
  8%|█████████▎                                                                                                       | 205/2499 [25:47<8:05:07, 12.69s/it]
  8%|█████████▎                                                                                                       | 206/2499 [25:53<6:51:31, 10.77s/it]
                                                                                                                                                           
{'loss': 0.626, 'grad_norm': 0.1439598947763443, 'learning_rate': 0.00019698633024874842, 'ppl': 1.8701, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4724.82, 'total_tokens': 9742266, 'epoch': 0.25}
-
  8%|█████████▎                                                                                                       | 206/2499 [25:53<6:51:31, 10.77s/it]
  8%|█████████▎                                                                                                       | 207/2499 [26:00<5:59:38,  9.41s/it]
                                                                                                                                                           
{'loss': 0.6171, 'grad_norm': 0.17815396189689636, 'learning_rate': 0.00019695549976315245, 'ppl': 1.8535, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4507.21, 'total_tokens': 9770456, 'epoch': 0.25}
-
  8%|█████████▎                                                                                                       | 207/2499 [26:00<5:59:38,  9.41s/it]
  8%|█████████▍                                                                                                       | 208/2499 [26:06<5:23:29,  8.47s/it]
                                                                                                                                                           
{'loss': 0.569, 'grad_norm': 0.15135987102985382, 'learning_rate': 0.0001969245148153111, 'ppl': 1.7665, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4686.34, 'total_tokens': 9799838, 'epoch': 0.25}
-
  8%|█████████▍                                                                                                       | 208/2499 [26:06<5:23:29,  8.47s/it]
  8%|█████████▍                                                                                                       | 209/2499 [26:12<4:58:00,  7.81s/it]
                                                                                                                                                           
{'loss': 0.6061, 'grad_norm': 0.15614871680736542, 'learning_rate': 0.00019689337545458727, 'ppl': 1.8333, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4564.09, 'total_tokens': 9828394, 'epoch': 0.25}
-
  8%|█████████▍                                                                                                       | 209/2499 [26:12<4:58:00,  7.81s/it]
  8%|█████████▍                                                                                                       | 210/2499 [26:18<4:40:23,  7.35s/it]
                                                                                                                                                           
{'loss': 0.5968, 'grad_norm': 0.155409038066864, 'learning_rate': 0.00019686208173058987, 'ppl': 1.8163, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4670.39, 'total_tokens': 9857717, 'epoch': 0.25}
-
  8%|█████████▍                                                                                                       | 210/2499 [26:18<4:40:23,  7.35s/it]
  8%|█████████▌                                                                                                       | 211/2499 [26:25<4:28:24,  7.04s/it]
                                                                                                                                                           
{'loss': 0.6103, 'grad_norm': 0.14444303512573242, 'learning_rate': 0.00019683063369317364, 'ppl': 1.841, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4549.36, 'total_tokens': 9886424, 'epoch': 0.25}
-
  8%|█████████▌                                                                                                       | 211/2499 [26:25<4:28:24,  7.04s/it]
  8%|█████████▌                                                                                                       | 212/2499 [26:31<4:19:41,  6.81s/it]
                                                                                                                                                           
{'loss': 0.6344, 'grad_norm': 0.15831312537193298, 'learning_rate': 0.00019679903139243937, 'ppl': 1.8859, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4649.2, 'total_tokens': 9915636, 'epoch': 0.25}
-
  8%|█████████▌                                                                                                       | 212/2499 [26:31<4:19:41,  6.81s/it]
  9%|█████████▋                                                                                                       | 213/2499 [26:37<4:13:23,  6.65s/it]
                                                                                                                                                           
{'loss': 0.6425, 'grad_norm': 0.16117224097251892, 'learning_rate': 0.00019676727487873334, 'ppl': 1.9012, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4553.31, 'total_tokens': 9944180, 'epoch': 0.26}
-
  9%|█████████▋                                                                                                       | 213/2499 [26:37<4:13:23,  6.65s/it]
  9%|█████████▋                                                                                                       | 214/2499 [26:44<4:09:02,  6.54s/it]
                                                                                                                                                           
{'loss': 0.6397, 'grad_norm': 0.15113097429275513, 'learning_rate': 0.00019673536420264774, 'ppl': 1.8959, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4723.13, 'total_tokens': 9973833, 'epoch': 0.26}
-
  9%|█████████▋                                                                                                       | 214/2499 [26:44<4:09:02,  6.54s/it]
  9%|█████████▋                                                                                                       | 215/2499 [26:50<4:05:51,  6.46s/it]
                                                                                                                                                           
{'loss': 0.6297, 'grad_norm': 0.16022402048110962, 'learning_rate': 0.00019670329941502023, 'ppl': 1.877, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4658.6, 'total_tokens': 10003030, 'epoch': 0.26}
-
  9%|█████████▋                                                                                                       | 215/2499 [26:50<4:05:51,  6.46s/it]
  9%|█████████▊                                                                                                       | 216/2499 [26:56<4:03:28,  6.40s/it]
                                                                                                                                                           
{'loss': 0.6056, 'grad_norm': 0.14694246649742126, 'learning_rate': 0.00019667108056693405, 'ppl': 1.8324, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4430.63, 'total_tokens': 10030750, 'epoch': 0.26}
-
  9%|█████████▊                                                                                                       | 216/2499 [26:56<4:03:28,  6.40s/it]
  9%|█████████▊                                                                                                       | 217/2499 [27:02<4:02:15,  6.37s/it]
                                                                                                                                                           
{'loss': 0.5919, 'grad_norm': 0.24986422061920166, 'learning_rate': 0.00019663870770971788, 'ppl': 1.8074, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4534.13, 'total_tokens': 10059315, 'epoch': 0.26}
-
  9%|█████████▊                                                                                                       | 217/2499 [27:02<4:02:15,  6.37s/it]
  9%|█████████▊                                                                                                       | 218/2499 [27:09<4:01:23,  6.35s/it]
                                                                                                                                                           
{'loss': 0.6979, 'grad_norm': 0.16677173972129822, 'learning_rate': 0.0001966061808949457, 'ppl': 2.0095, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4392.88, 'total_tokens': 10086983, 'epoch': 0.26}
-
  9%|█████████▊                                                                                                       | 218/2499 [27:09<4:01:23,  6.35s/it]
  9%|█████████▉                                                                                                       | 219/2499 [27:15<4:00:25,  6.33s/it]
                                                                                                                                                           
{'loss': 0.6049, 'grad_norm': 0.15127292275428772, 'learning_rate': 0.0001965735001744368, 'ppl': 1.8311, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4572.27, 'total_tokens': 10115652, 'epoch': 0.26}
-
  9%|█████████▉                                                                                                       | 219/2499 [27:15<4:00:25,  6.33s/it]
  9%|█████████▉                                                                                                       | 220/2499 [27:21<3:59:42,  6.31s/it]
                                                                                                                                                           
{'loss': 0.6062, 'grad_norm': 0.15187977254390717, 'learning_rate': 0.00019654066560025567, 'ppl': 1.8335, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4528.19, 'total_tokens': 10144050, 'epoch': 0.26}
-
  9%|█████████▉                                                                                                       | 220/2499 [27:21<3:59:42,  6.31s/it]
  9%|█████████▉                                                                                                       | 221/2499 [27:27<3:59:15,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5865, 'grad_norm': 0.15046511590480804, 'learning_rate': 0.00019650767722471186, 'ppl': 1.7977, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4610.57, 'total_tokens': 10172997, 'epoch': 0.27}
-
  9%|█████████▉                                                                                                       | 221/2499 [27:27<3:59:15,  6.30s/it]
  9%|██████████                                                                                                       | 222/2499 [27:34<3:58:50,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5561, 'grad_norm': 0.138535276055336, 'learning_rate': 0.00019647453510036002, 'ppl': 1.7439, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4535.14, 'total_tokens': 10201441, 'epoch': 0.27}
-
  9%|██████████                                                                                                       | 222/2499 [27:34<3:58:50,  6.29s/it]
  9%|██████████                                                                                                       | 223/2499 [27:40<3:58:42,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6202, 'grad_norm': 0.15157605707645416, 'learning_rate': 0.0001964412392799997, 'ppl': 1.8593, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4750.1, 'total_tokens': 10231316, 'epoch': 0.27}
-
  9%|██████████                                                                                                       | 223/2499 [27:40<3:58:42,  6.29s/it]
  9%|██████████▏                                                                                                      | 224/2499 [27:46<3:58:53,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6144, 'grad_norm': 0.15931400656700134, 'learning_rate': 0.00019640778981667526, 'ppl': 1.8485, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4641.35, 'total_tokens': 10260630, 'epoch': 0.27}
-
  9%|██████████▏                                                                                                      | 224/2499 [27:46<3:58:53,  6.30s/it]
  9%|██████████▏                                                                                                      | 225/2499 [27:53<3:58:56,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6279, 'grad_norm': 0.1776997298002243, 'learning_rate': 0.00019637418676367595, 'ppl': 1.8737, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4354.47, 'total_tokens': 10288107, 'epoch': 0.27}
-
  9%|██████████▏                                                                                                      | 225/2499 [27:53<3:58:56,  6.30s/it]
  9%|██████████▏                                                                                                      | 226/2499 [27:59<3:58:40,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6077, 'grad_norm': 0.15269804000854492, 'learning_rate': 0.00019634043017453565, 'ppl': 1.8362, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4585.03, 'total_tokens': 10316931, 'epoch': 0.27}
-
  9%|██████████▏                                                                                                      | 226/2499 [27:59<3:58:40,  6.30s/it]
  9%|██████████▎                                                                                                      | 227/2499 [28:05<3:58:11,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6055, 'grad_norm': 0.15620221197605133, 'learning_rate': 0.00019630652010303273, 'ppl': 1.8322, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4495.93, 'total_tokens': 10345097, 'epoch': 0.27}
-
  9%|██████████▎                                                                                                      | 227/2499 [28:05<3:58:11,  6.29s/it]
  9%|██████████▎                                                                                                      | 228/2499 [28:12<3:57:59,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6165, 'grad_norm': 0.1660533845424652, 'learning_rate': 0.00019627245660319026, 'ppl': 1.8524, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4621.54, 'total_tokens': 10374119, 'epoch': 0.27}
-
  9%|██████████▎                                                                                                      | 228/2499 [28:12<3:57:59,  6.29s/it]
  9%|██████████▎                                                                                                      | 229/2499 [28:18<3:58:06,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6691, 'grad_norm': 0.17098499834537506, 'learning_rate': 0.00019623823972927563, 'ppl': 1.9525, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4424.1, 'total_tokens': 10402014, 'epoch': 0.27}
-
  9%|██████████▎                                                                                                      | 229/2499 [28:18<3:58:06,  6.29s/it]
  9%|██████████▍                                                                                                      | 230/2499 [28:24<3:58:23,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5747, 'grad_norm': 0.15090154111385345, 'learning_rate': 0.00019620386953580067, 'ppl': 1.7766, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4648.9, 'total_tokens': 10431409, 'epoch': 0.28}
-
  9%|██████████▍                                                                                                      | 230/2499 [28:24<3:58:23,  6.30s/it]
  9%|██████████▍                                                                                                      | 231/2499 [28:30<3:58:22,  6.31s/it]
                                                                                                                                                           
{'loss': 0.6935, 'grad_norm': 0.1816420555114746, 'learning_rate': 0.00019616934607752127, 'ppl': 2.0007, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4424.06, 'total_tokens': 10459322, 'epoch': 0.28}
-
  9%|██████████▍                                                                                                      | 231/2499 [28:30<3:58:22,  6.31s/it]
  9%|██████████▍                                                                                                      | 232/2499 [28:37<3:58:24,  6.31s/it]
                                                                                                                                                           
{'loss': 0.626, 'grad_norm': 0.1661038100719452, 'learning_rate': 0.0001961346694094377, 'ppl': 1.8701, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4496.08, 'total_tokens': 10487718, 'epoch': 0.28}
-
  9%|██████████▍                                                                                                      | 232/2499 [28:37<3:58:24,  6.31s/it]
  9%|██████████▌                                                                                                      | 233/2499 [28:43<3:58:03,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6463, 'grad_norm': 0.1440490186214447, 'learning_rate': 0.00019609983958679422, 'ppl': 1.9085, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4682.72, 'total_tokens': 10517146, 'epoch': 0.28}
-
  9%|██████████▌                                                                                                      | 233/2499 [28:43<3:58:03,  6.30s/it]
  9%|██████████▌                                                                                                      | 234/2499 [28:49<3:57:34,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6234, 'grad_norm': 0.1468016356229782, 'learning_rate': 0.00019606485666507912, 'ppl': 1.8653, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4655.6, 'total_tokens': 10546326, 'epoch': 0.28}
-
  9%|██████████▌                                                                                                      | 234/2499 [28:49<3:57:34,  6.29s/it]
  9%|██████████▋                                                                                                      | 235/2499 [28:56<3:57:11,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5476, 'grad_norm': 0.15022438764572144, 'learning_rate': 0.00019602972070002452, 'ppl': 1.7291, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4583.31, 'total_tokens': 10575047, 'epoch': 0.28}
-
  9%|██████████▋                                                                                                      | 235/2499 [28:56<3:57:11,  6.29s/it]
  9%|██████████▋                                                                                                      | 236/2499 [29:02<3:56:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5992, 'grad_norm': 0.15142318606376648, 'learning_rate': 0.00019599443174760648, 'ppl': 1.8207, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4589.7, 'total_tokens': 10603836, 'epoch': 0.28}
-
  9%|██████████▋                                                                                                      | 236/2499 [29:02<3:56:57,  6.28s/it]
  9%|██████████▋                                                                                                      | 237/2499 [29:08<3:56:44,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6017, 'grad_norm': 0.14540834724903107, 'learning_rate': 0.00019595898986404464, 'ppl': 1.8252, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4538.75, 'total_tokens': 10632296, 'epoch': 0.28}
-
  9%|██████████▋                                                                                                      | 237/2499 [29:08<3:56:44,  6.28s/it]
 10%|██████████▊                                                                                                      | 238/2499 [29:14<3:57:00,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6406, 'grad_norm': 0.16254767775535583, 'learning_rate': 0.00019592339510580243, 'ppl': 1.8976, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4589.97, 'total_tokens': 10661260, 'epoch': 0.29}
-
 10%|██████████▊                                                                                                      | 238/2499 [29:14<3:57:00,  6.29s/it]
 10%|██████████▊                                                                                                      | 239/2499 [29:21<3:57:02,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6249, 'grad_norm': 0.14255541563034058, 'learning_rate': 0.00019588764752958668, 'ppl': 1.8681, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4737.33, 'total_tokens': 10691094, 'epoch': 0.29}
-
 10%|██████████▊                                                                                                      | 239/2499 [29:21<3:57:02,  6.29s/it]
 10%|██████████▊                                                                                                      | 240/2499 [29:27<3:56:41,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6637, 'grad_norm': 0.1646222472190857, 'learning_rate': 0.00019585174719234782, 'ppl': 1.942, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4616.52, 'total_tokens': 10720034, 'epoch': 0.29}
-
 10%|██████████▊                                                                                                      | 240/2499 [29:27<3:56:41,  6.29s/it]
 10%|██████████▉                                                                                                      | 241/2499 [29:33<3:56:15,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5695, 'grad_norm': 0.14777596294879913, 'learning_rate': 0.00019581569415127954, 'ppl': 1.7674, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4495.8, 'total_tokens': 10748161, 'epoch': 0.29}
-
 10%|██████████▉                                                                                                      | 241/2499 [29:33<3:56:15,  6.28s/it]
 10%|██████████▉                                                                                                      | 242/2499 [29:40<3:55:55,  6.27s/it]
                                                                                                                                                           
{'loss': 0.629, 'grad_norm': 0.17004624009132385, 'learning_rate': 0.00019577948846381886, 'ppl': 1.8757, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4432.93, 'total_tokens': 10775888, 'epoch': 0.29}
-
 10%|██████████▉                                                                                                      | 242/2499 [29:40<3:55:55,  6.27s/it]
 10%|██████████▉                                                                                                      | 243/2499 [29:46<3:55:47,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6156, 'grad_norm': 0.15959785878658295, 'learning_rate': 0.00019574313018764597, 'ppl': 1.8508, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4592.21, 'total_tokens': 10804671, 'epoch': 0.29}
-
 10%|██████████▉                                                                                                      | 243/2499 [29:46<3:55:47,  6.27s/it]
 10%|█████████��█                                                                                                      | 244/2499 [29:52<3:55:42,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6562, 'grad_norm': 0.15434452891349792, 'learning_rate': 0.00019570661938068417, 'ppl': 1.9275, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4657.16, 'total_tokens': 10833870, 'epoch': 0.29}
-
 10%|███████████                                                                                                      | 244/2499 [29:52<3:55:42,  6.27s/it]
 10%|███████████                                                                                                      | 245/2499 [29:58<3:55:51,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6261, 'grad_norm': 0.15079908072948456, 'learning_rate': 0.00019566995610109967, 'ppl': 1.8703, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4408.02, 'total_tokens': 10861605, 'epoch': 0.29}
-
 10%|███████████                                                                                                      | 245/2499 [29:58<3:55:51,  6.28s/it]
 10%|███████████                                                                                                      | 246/2499 [30:05<3:55:38,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6186, 'grad_norm': 0.15629984438419342, 'learning_rate': 0.0001956331404073018, 'ppl': 1.8563, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4470.79, 'total_tokens': 10889613, 'epoch': 0.3}
-
 10%|███████████                                                                                                      | 246/2499 [30:05<3:55:38,  6.28s/it]
 10%|███████████▏                                                                                                     | 247/2499 [30:11<3:55:48,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6016, 'grad_norm': 0.1890110820531845, 'learning_rate': 0.00019559617235794243, 'ppl': 1.825, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4523.88, 'total_tokens': 10918100, 'epoch': 0.3}
-
 10%|███████████▏                                                                                                     | 247/2499 [30:11<3:55:48,  6.28s/it]
 10%|███████████▏                                                                                                     | 248/2499 [30:17<3:55:27,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5688, 'grad_norm': 0.15975165367126465, 'learning_rate': 0.00019555905201191638, 'ppl': 1.7661, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4415.02, 'total_tokens': 10945730, 'epoch': 0.3}
-
 10%|███████████▏                                                                                                     | 248/2499 [30:17<3:55:27,  6.28s/it]
 10%|███████████▎                                                                                                     | 249/2499 [30:24<3:55:16,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6438, 'grad_norm': 0.17490845918655396, 'learning_rate': 0.00019552177942836098, 'ppl': 1.9037, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4577.94, 'total_tokens': 10974419, 'epoch': 0.3}
-
 10%|███████████▎                                                                                                     | 249/2499 [30:24<3:55:16,  6.27s/it]
 10%|███████████▎                                                                                                     | 250/2499 [30:30<3:55:24,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6401, 'grad_norm': 0.1564694046974182, 'learning_rate': 0.0001954843546666561, 'ppl': 1.8967, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4774.51, 'total_tokens': 11004462, 'epoch': 0.3}
-
 10%|███████████▎                                                                                                     | 250/2499 [30:30<3:55:24,  6.28s/it]
 10%|███████████▎                                                                                                     | 251/2499 [30:36<3:55:28,  6.28s/it]
                                                                                                                                                           
{'loss': 0.572, 'grad_norm': 0.15054045617580414, 'learning_rate': 0.00019544677778642408, 'ppl': 1.7718, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4414.85, 'total_tokens': 11032247, 'epoch': 0.3}
-
 10%|███████████▎                                                                                                     | 251/2499 [30:36<3:55:28,  6.28s/it]
 10%|███████████▍                                                                                                     | 252/2499 [30:42<3:55:48,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5845, 'grad_norm': 0.15074437856674194, 'learning_rate': 0.00019540904884752963, 'ppl': 1.7941, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4519.84, 'total_tokens': 11060814, 'epoch': 0.3}
-
 10%|███████████▍                                                                                                     | 252/2499 [30:42<3:55:48,  6.30s/it]
 10%|███████████▍                                                                                                     | 253/2499 [30:49<3:55:41,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6189, 'grad_norm': 0.17772871255874634, 'learning_rate': 0.0001953711679100796, 'ppl': 1.8569, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4681.42, 'total_tokens': 11090268, 'epoch': 0.3}
-
 10%|███████████▍                                                                                                     | 253/2499 [30:49<3:55:41,  6.30s/it]
 10%|███████████▍                                                                                                     | 254/2499 [30:55<3:55:17,  6.29s/it]
                                                                                                                                                           
{'loss': 0.636, 'grad_norm': 0.16250668466091156, 'learning_rate': 0.00019533313503442313, 'ppl': 1.8889, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4665.44, 'total_tokens': 11119505, 'epoch': 0.3}
-
 10%|███████████▍                                                                                                     | 254/2499 [30:55<3:55:17,  6.29s/it]
 10%|███████████▌                                                                                                     | 255/2499 [31:01<3:54:59,  6.28s/it]
                                                                                                                                                           
{'loss': 0.617, 'grad_norm': 0.15999537706375122, 'learning_rate': 0.0001952949502811513, 'ppl': 1.8534, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4580.22, 'total_tokens': 11148217, 'epoch': 0.31}
-
 10%|███████████▌                                                                                                     | 255/2499 [31:01<3:54:59,  6.28s/it]
 10%|███████████▌                                                                                                     | 256/2499 [31:08<3:54:43,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6121, 'grad_norm': 0.1519382745027542, 'learning_rate': 0.00019525661371109726, 'ppl': 1.8443, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4643.22, 'total_tokens': 11177318, 'epoch': 0.31}
-
 10%|███████████▌                                                                                                     | 256/2499 [31:08<3:54:43,  6.28s/it]
 10%|███████████▌                                                                                                     | 257/2499 [31:14<3:54:29,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6486, 'grad_norm': 0.16151119768619537, 'learning_rate': 0.00019521812538533592, 'ppl': 1.9129, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4493.36, 'total_tokens': 11205465, 'epoch': 0.31}
-
 10%|███████████▌                                                                                                     | 257/2499 [31:14<3:54:29,  6.28s/it]
 10%|███████████▋                                                                                                     | 258/2499 [31:20<3:54:42,  6.28s/it]
                                                                                                                                                           
{'loss': 0.602, 'grad_norm': 0.15068507194519043, 'learning_rate': 0.000195179485365184, 'ppl': 1.8258, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4588.52, 'total_tokens': 11234379, 'epoch': 0.31}
-
 10%|███████████▋                                                                                                     | 258/2499 [31:20<3:54:42,  6.28s/it]
 10%|███████████▋                                                                                                     | 259/2499 [31:26<3:55:04,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5858, 'grad_norm': 0.14763666689395905, 'learning_rate': 0.00019514069371219989, 'ppl': 1.7964, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4632.62, 'total_tokens': 11263667, 'epoch': 0.31}
-
 10%|███████████▋                                                                                                     | 259/2499 [31:26<3:55:04,  6.30s/it]
 10%|███████████▊                                                                                                     | 260/2499 [31:33<3:54:40,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5806, 'grad_norm': 0.1620749682188034, 'learning_rate': 0.00019510175048818358, 'ppl': 1.7871, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4537.22, 'total_tokens': 11292098, 'epoch': 0.31}
-
 10%|███████████▊                                                                                                     | 260/2499 [31:33<3:54:40,  6.29s/it]
 10%|███████████▊                                                                                                     | 261/2499 [31:39<3:54:14,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6139, 'grad_norm': 0.16305935382843018, 'learning_rate': 0.00019506265575517645, 'ppl': 1.8476, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4553.53, 'total_tokens': 11320589, 'epoch': 0.31}
-
 10%|███████████▊                                                                                                     | 261/2499 [31:39<3:54:14,  6.28s/it]
 10%|███████████▊                                                                                                     | 262/2499 [31:45<3:53:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5836, 'grad_norm': 0.1518191397190094, 'learning_rate': 0.00019502340957546133, 'ppl': 1.7925, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4476.69, 'total_tokens': 11348623, 'epoch': 0.31}
-
 10%|███████████▊                                                                                                     | 262/2499 [31:45<3:53:57,  6.28s/it]
 11%|███████████▉                                                                                                     | 263/2499 [31:52<3:54:00,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6227, 'grad_norm': 0.16224627196788788, 'learning_rate': 0.00019498401201156233, 'ppl': 1.864, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4719.83, 'total_tokens': 11378293, 'epoch': 0.32}
-
 11%|███████████▉                                                                                                     | 263/2499 [31:52<3:54:00,  6.28s/it]
 11%|███████████▉                                                                                                     | 264/2499 [31:58<3:53:56,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5515, 'grad_norm': 0.15429292619228363, 'learning_rate': 0.00019494446312624464, 'ppl': 1.7359, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4659.78, 'total_tokens': 11407559, 'epoch': 0.32}
-
 11%|███████████▉                                                                                                     | 264/2499 [31:58<3:53:56,  6.28s/it]
 11%|███████████▉                                                                                                     | 265/2499 [32:04<3:54:07,  6.29s/it]
                                                                                                                                                           
{'loss': 0.627, 'grad_norm': 0.14916737377643585, 'learning_rate': 0.0001949047629825146, 'ppl': 1.872, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4431.08, 'total_tokens': 11435494, 'epoch': 0.32}
-
 11%|███████████▉                                                                                                     | 265/2499 [32:04<3:54:07,  6.29s/it]
 11%|████████████                                                                                                     | 266/2499 [32:10<3:54:24,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6209, 'grad_norm': 0.16902928054332733, 'learning_rate': 0.00019486491164361953, 'ppl': 1.8606, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4605.07, 'total_tokens': 11464595, 'epoch': 0.32}
-
 11%|████████████                                                                                                     | 266/2499 [32:10<3:54:24,  6.30s/it]
 11%|████████████                                                                                                     | 267/2499 [32:17<3:54:07,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5865, 'grad_norm': 0.17003753781318665, 'learning_rate': 0.00019482490917304757, 'ppl': 1.7977, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4610.91, 'total_tokens': 11493546, 'epoch': 0.32}
-
 11%|████████████                                                                                                     | 267/2499 [32:17<3:54:07,  6.29s/it]
 11%|████████████                                                                                                     | 268/2499 [32:23<3:53:43,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6035, 'grad_norm': 0.14946606755256653, 'learning_rate': 0.0001947847556345277, 'ppl': 1.8285, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4647.76, 'total_tokens': 11522665, 'epoch': 0.32}
-
 11%|████████████                                                                                                     | 268/2499 [32:23<3:53:43,  6.29s/it]
 11%|████████████▏                                                                                                    | 269/2499 [32:29<3:53:22,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6196, 'grad_norm': 0.1707615703344345, 'learning_rate': 0.0001947444510920295, 'ppl': 1.8582, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4341.34, 'total_tokens': 11549847, 'epoch': 0.32}
-
 11%|████████████▏                                                                                                    | 269/2499 [32:29<3:53:22,  6.28s/it]
 11%|████████████▏                                                                                                    | 270/2499 [32:36<3:53:28,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5874, 'grad_norm': 0.16345329582691193, 'learning_rate': 0.00019470399560976315, 'ppl': 1.7993, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4518.74, 'total_tokens': 11578294, 'epoch': 0.32}
-
 11%|████████████▏                                                                                                    | 270/2499 [32:36<3:53:28,  6.28s/it]
 11%|████████████▎                                                                                                    | 271/2499 [32:42<3:52:54,  6.27s/it]
                                                                                                                                                           
{'loss': 0.571, 'grad_norm': 0.162063866853714, 'learning_rate': 0.00019466338925217932, 'ppl': 1.77, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4312.6, 'total_tokens': 11605204, 'epoch': 0.33}
-
 11%|████████████▎                                                                                                    | 271/2499 [32:42<3:52:54,  6.27s/it]
 11%|████████████▎                                                                                                    | 272/2499 [32:48<3:53:16,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6094, 'grad_norm': 0.16176359355449677, 'learning_rate': 0.000194622632083969, 'ppl': 1.8393, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4681.13, 'total_tokens': 11634753, 'epoch': 0.33}
-
 11%|████████████▎                                                                                                    | 272/2499 [32:48<3:53:16,  6.28s/it]
 11%|████████████▎                                                                                                    | 273/2499 [32:54<3:53:29,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6469, 'grad_norm': 0.1526668667793274, 'learning_rate': 0.00019458172417006347, 'ppl': 1.9096, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4609.18, 'total_tokens': 11663837, 'epoch': 0.33}
-
 11%|████████████▎                                                                                                    | 273/2499 [32:54<3:53:29,  6.29s/it]
 11%|████████████▍                                                                                                    | 274/2499 [33:01<3:53:07,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6463, 'grad_norm': 0.17007103562355042, 'learning_rate': 0.00019454066557563415, 'ppl': 1.9085, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4541.62, 'total_tokens': 11692294, 'epoch': 0.33}
-
 11%|████████████▍                                                                                                    | 274/2499 [33:01<3:53:07,  6.29s/it]
 11%|████████████▍                                                                                                    | 275/2499 [33:07<3:52:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6217, 'grad_norm': 0.16375601291656494, 'learning_rate': 0.00019449945636609254, 'ppl': 1.8621, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4667.04, 'total_tokens': 11721595, 'epoch': 0.33}
-
 11%|████████████▍                                                                                                    | 275/2499 [33:07<3:52:57,  6.28s/it]
 11%|████████████▍                                                                                                    | 276/2499 [33:13<3:52:39,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6596, 'grad_norm': 0.16178229451179504, 'learning_rate': 0.00019445809660709006, 'ppl': 1.934, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4510.52, 'total_tokens': 11749853, 'epoch': 0.33}
-
 11%|████████████▍                                                                                                    | 276/2499 [33:13<3:52:39,  6.28s/it]
 11%|████████████▌                                                                                                    | 277/2499 [33:19<3:52:24,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6449, 'grad_norm': 0.17449264228343964, 'learning_rate': 0.00019441658636451794, 'ppl': 1.9058, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4589.89, 'total_tokens': 11778605, 'epoch': 0.33}
-
 11%|████████████▌                                                                                                    | 277/2499 [33:19<3:52:24,  6.28s/it]
 11%|████████████▌                                                                                                    | 278/2499 [33:26<3:52:22,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6853, 'grad_norm': 0.15174663066864014, 'learning_rate': 0.00019437492570450725, 'ppl': 1.9844, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4658.29, 'total_tokens': 11807861, 'epoch': 0.33}
-
 11%|████████████▌                                                                                                    | 278/2499 [33:26<3:52:22,  6.28s/it]
 11%|████████████▌                                                                                                    | 279/2499 [33:32<3:52:19,  6.28s/it]
                                                                                                                                                           
{'loss': 0.584, 'grad_norm': 0.15343217551708221, 'learning_rate': 0.0001943331146934286, 'ppl': 1.7932, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4281.23, 'total_tokens': 11834747, 'epoch': 0.33}
-
 11%|████████████▌                                                                                                    | 279/2499 [33:32<3:52:19,  6.28s/it]
 11%|████████████▋                                                                                                    | 280/2499 [33:38<3:52:13,  6.28s/it]
                                                                                                                                                           
{'loss': 0.622, 'grad_norm': 0.1785060465335846, 'learning_rate': 0.00019429115339789215, 'ppl': 1.8626, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4445.31, 'total_tokens': 11862643, 'epoch': 0.34}
-
 11%|████████████▋                                                                                                    | 280/2499 [33:38<3:52:13,  6.28s/it]
 11%|████████████▋                                                                                                    | 281/2499 [33:45<3:52:09,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5981, 'grad_norm': 0.1548158824443817, 'learning_rate': 0.00019424904188474753, 'ppl': 1.8187, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4637.88, 'total_tokens': 11891771, 'epoch': 0.34}
-
 11%|████████████▋                                                                                                    | 281/2499 [33:45<3:52:09,  6.28s/it]
 11%|████████████▊                                                                                                    | 282/2499 [33:51<3:52:02,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5534, 'grad_norm': 0.1484275758266449, 'learning_rate': 0.00019420678022108364, 'ppl': 1.7392, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4680.22, 'total_tokens': 11921146, 'epoch': 0.34}
-
 11%|████████████▊                                                                                                    | 282/2499 [33:51<3:52:02,  6.28s/it]
 11%|████████████▊                                                                                                    | 283/2499 [33:57<3:51:41,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6026, 'grad_norm': 0.17198914289474487, 'learning_rate': 0.00019416436847422862, 'ppl': 1.8269, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4457.61, 'total_tokens': 11949033, 'epoch': 0.34}
-
 11%|████████████▊                                                                                                    | 283/2499 [33:57<3:51:41,  6.27s/it]
 11%|████████████▊                                                                                                    | 284/2499 [34:03<3:51:36,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5981, 'grad_norm': 0.16630032658576965, 'learning_rate': 0.00019412180671174965, 'ppl': 1.8187, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4588.48, 'total_tokens': 11977816, 'epoch': 0.34}
-
 11%|████████████▊                                                                                                    | 284/2499 [34:03<3:51:36,  6.27s/it]
 11%|████████████▉                                                                                                    | 285/2499 [34:10<3:51:48,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5998, 'grad_norm': 0.15046648681163788, 'learning_rate': 0.00019407909500145302, 'ppl': 1.8218, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4489.1, 'total_tokens': 12006094, 'epoch': 0.34}
-
 11%|████████████▉                                                                                                    | 285/2499 [34:10<3:51:48,  6.28s/it]
 11%|████████████▉                                                                                                    | 286/2499 [34:16<3:52:13,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6113, 'grad_norm': 0.1513357311487198, 'learning_rate': 0.0001940362334113837, 'ppl': 1.8428, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4664.7, 'total_tokens': 12035602, 'epoch': 0.34}
-
 11%|████████████▉                                                                                                    | 286/2499 [34:16<3:52:13,  6.30s/it]
 11%|████████████▉                                                                                                    | 287/2499 [34:22<3:51:53,  6.29s/it]
                                                                                                                                                           
{'loss': 0.635, 'grad_norm': 0.15904900431632996, 'learning_rate': 0.00019399322200982567, 'ppl': 1.887, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4445.6, 'total_tokens': 12063481, 'epoch': 0.34}
-
 11%|████████████▉                                                                                                    | 287/2499 [34:22<3:51:53,  6.29s/it]
 12%|█████████████                                                                                                    | 288/2499 [34:29<3:51:35,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5955, 'grad_norm': 0.16054871678352356, 'learning_rate': 0.00019395006086530146, 'ppl': 1.8139, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4601.21, 'total_tokens': 12092331, 'epoch': 0.35}
-
 12%|█████████████                                                                                                    | 288/2499 [34:29<3:51:35,  6.28s/it]
 12%|█████████████                                                                                                    | 289/2499 [34:35<3:51:23,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5774, 'grad_norm': 0.16297683119773865, 'learning_rate': 0.00019390675004657214, 'ppl': 1.7814, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4571.41, 'total_tokens': 12121009, 'epoch': 0.35}
-
 12%|█████████████                                                                                                    | 289/2499 [34:35<3:51:23,  6.28s/it]
 12%|█████████████                                                                                                    | 290/2499 [34:41<3:51:09,  6.28s/it]
                                                                                                                                                           
{'loss': 0.617, 'grad_norm': 0.17384764552116394, 'learning_rate': 0.0001938632896226373, 'ppl': 1.8534, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4560.04, 'total_tokens': 12149596, 'epoch': 0.35}
-
 12%|█████████████                                                                                                    | 290/2499 [34:41<3:51:09,  6.28s/it]
 12%|█████████████▏                                                                                                   | 291/2499 [34:47<3:50:58,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6401, 'grad_norm': 0.1762438863515854, 'learning_rate': 0.00019381967966273483, 'ppl': 1.8967, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4557.77, 'total_tokens': 12178171, 'epoch': 0.35}
-
 12%|█████████████▏                                                                                                   | 291/2499 [34:47<3:50:58,  6.28s/it]
 12%|█████████████▏                                                                                                   | 292/2499 [34:54<3:51:08,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6359, 'grad_norm': 0.18544957041740417, 'learning_rate': 0.0001937759202363408, 'ppl': 1.8887, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4622.3, 'total_tokens': 12207284, 'epoch': 0.35}
-
 12%|█████████████▏                                                                                                   | 292/2499 [34:54<3:51:08,  6.28s/it]
 12%|█████████████▏                                                                                                   | 293/2499 [35:00<3:51:26,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5587, 'grad_norm': 0.14742697775363922, 'learning_rate': 0.00019373201141316954, 'ppl': 1.7484, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4573.06, 'total_tokens': 12236170, 'epoch': 0.35}
-
 12%|█████████████▏                                                                                                   | 293/2499 [35:00<3:51:26,  6.29s/it]
 12%|█████████████▎                                                                                                   | 294/2499 [35:06<3:51:16,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6296, 'grad_norm': 0.16413316130638123, 'learning_rate': 0.00019368795326317326, 'ppl': 1.8769, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4740.96, 'total_tokens': 12265971, 'epoch': 0.35}
-
 12%|█████████████▎                                                                                                   | 294/2499 [35:06<3:51:16,  6.29s/it]
 12%|█████████████▎                                                                                                   | 295/2499 [35:13<3:51:01,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5648, 'grad_norm': 0.14326909184455872, 'learning_rate': 0.0001936437458565421, 'ppl': 1.7591, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4746.91, 'total_tokens': 12295770, 'epoch': 0.35}
-
 12%|█████████████▎                                                                                                   | 295/2499 [35:13<3:51:01,  6.29s/it]
 12%|█████████████▍                                                                                                   | 296/2499 [35:19<3:50:57,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5588, 'grad_norm': 0.14970144629478455, 'learning_rate': 0.000193599389263704, 'ppl': 1.7486, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4777.71, 'total_tokens': 12325825, 'epoch': 0.36}
-
 12%|█████████████▍                                                                                                   | 296/2499 [35:19<3:50:57,  6.29s/it]
 12%|█████████████▍                                                                                                   | 297/2499 [35:25<3:50:28,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5853, 'grad_norm': 0.15977391600608826, 'learning_rate': 0.00019355488355532455, 'ppl': 1.7955, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4499.97, 'total_tokens': 12353968, 'epoch': 0.36}
-
 12%|█████████████▍                                                                                                   | 297/2499 [35:25<3:50:28,  6.28s/it]
 12%|█████████████▍                                                                                                   | 298/2499 [35:31<3:50:17,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5913, 'grad_norm': 0.15226499736309052, 'learning_rate': 0.00019351022880230694, 'ppl': 1.8063, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4561.15, 'total_tokens': 12382567, 'epoch': 0.36}
-
 12%|█████████████▍                                                                                                   | 298/2499 [35:31<3:50:17,  6.28s/it]
 12%|█████████████▌                                                                                                   | 299/2499 [35:38<3:50:33,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5725, 'grad_norm': 0.14446371793746948, 'learning_rate': 0.00019346542507579176, 'ppl': 1.7727, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4527.6, 'total_tokens': 12411134, 'epoch': 0.36}
-
 12%|█████████████▌                                                                                                   | 299/2499 [35:38<3:50:33,  6.29s/it]
 12%|█████████████▌                                                                                                   | 300/2499 [35:44<3:50:45,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6065, 'grad_norm': 0.1595160961151123, 'learning_rate': 0.00019342047244715695, 'ppl': 1.834, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4570.99, 'total_tokens': 12439985, 'epoch': 0.36}
-
 12%|█████████████▌                                                                                                   | 300/2499 [35:44<3:50:45,  6.30s/it][2025-12-28 11:41:20,692] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 11:41:22,581] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8238303661346436
-[2025-12-28 11:41:23,449] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8675413131713867
-[2025-12-28 11:41:24,254] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.804617166519165
-[2025-12-28 11:41:25,042] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.7866499423980713
-[2025-12-28 11:41:25,042] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
-
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.41it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:54,  1.58it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<01:00,  1.43it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:05,  1.30it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:05,  1.27it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:08,  1.22it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:06<01:07,  1.22it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:08,  1.18it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:06,  1.20it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:07,  1.17it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:05,  1.18it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:06,  1.16it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:04,  1.18it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:04,  1.16it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:12<01:02,  1.18it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:13<01:03,  1.16it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:01,  1.18it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:01,  1.16it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<00:59,  1.17it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<00:59,  1.16it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:57,  1.18it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:18<00:57,  1.16it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:19<00:56,  1.18it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:56,  1.15it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.18it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:54,  1.15it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:52,  1.17it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:52,  1.15it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:24<00:51,  1.17it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:25<00:51,  1.15it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.17it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:54,  1.05it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:47,  1.17it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:48,  1.13it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:46,  1.17it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:46,  1.13it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:31<00:44,  1.17it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:32<00:44,  1.13it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:42,  1.17it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:43,  1.14it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:41,  1.17it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:41,  1.12it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:39,  1.16it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:38<00:39,  1.14it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:38<00:37,  1.17it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.15it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:36,  1.16it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.15it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:34,  1.17it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:34,  1.14it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:44<00:32,  1.17it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:44<00:32,  1.15it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.17it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.15it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:29,  1.17it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.15it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.17it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:27,  1.14it/s]
-
 67%|█████��████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:50<00:25,  1.16it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:51<00:25,  1.14it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:24,  1.16it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:53<00:23,  1.15it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:22,  1.17it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:21,  1.15it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:20,  1.17it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:57<00:20,  1.14it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:57<00:18,  1.16it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:58<00:18,  1.15it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.17it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.15it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.17it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.15it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.17it/s]
-
 83%|█████████████████████████���████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:03<00:13,  1.15it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:04<00:11,  1.17it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:05<00:11,  1.16it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.17it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.16it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.18it/s]
-
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 81/90 [01:09<00:07,  1.16it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:09<00:06,  1.18it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:10<00:06,  1.13it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:11<00:05,  1.17it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.14it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.17it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.14it/s]
-
 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████���████▍  | 88/90 [01:15<00:01,  1.17it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89/90 [01:16<00:00,  1.14it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:16<00:00,  1.14it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.5875148177146912, 'eval_runtime': 79.0999, 'eval_samples_per_second': 9.229, 'eval_steps_per_second': 2.314, 'eval_ppl': 1.7995, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.15, 'epoch': 0.36}
-
 12%|█████████████▌                                                                                                   | 300/2499 [37:08<3:50:45,  6.30s/it]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
-
                                                                                                                                                           
 12%|█████████████▍                                                                                                  | 301/2499 [37:14<19:08:11, 31.34s/it]
                                                                                                                                                           
{'loss': 0.5979, 'grad_norm': 0.16208259761333466, 'learning_rate': 0.00019337537098801767, 'ppl': 1.8183, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.11, 'tokens_per_second_per_gpu': 4645.22, 'total_tokens': 13739099, 'epoch': 0.36}
-
 12%|█████████████▍                                                                                                  | 301/2499 [37:14<19:08:11, 31.34s/it]
 12%|█████████████▌                                                                                                  | 302/2499 [37:20<14:32:27, 23.83s/it]
                                                                                                                                                           
{'loss': 0.5843, 'grad_norm': 0.16218189895153046, 'learning_rate': 0.00019333012077022622, 'ppl': 1.7937, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4714.28, 'total_tokens': 13768739, 'epoch': 0.36}
-
 12%|█████████████▌                                                                                                  | 302/2499 [37:20<14:32:27, 23.83s/it]
 12%|█████████████▌                                                                                                  | 303/2499 [37:26<11:19:37, 18.57s/it]
                                                                                                                                                           
{'loss': 0.584, 'grad_norm': 0.154547318816185, 'learning_rate': 0.00019328472186587185, 'ppl': 1.7932, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4635.54, 'total_tokens': 13797936, 'epoch': 0.36}
-
 12%|█████████████▌                                                                                                  | 303/2499 [37:26<11:19:37, 18.57s/it]
 12%|█████████████▋                                                                                                   | 304/2499 [37:33<9:04:39, 14.89s/it]
                                                                                                                                                           
{'loss': 0.6095, 'grad_norm': 0.17508265376091003, 'learning_rate': 0.0001932391743472807, 'ppl': 1.8395, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4645.04, 'total_tokens': 13827176, 'epoch': 0.36}
-
 12%|█████████████▋                                                                                                   | 304/2499 [37:33<9:04:39, 14.89s/it]
 12%|█████████████▊                                                                                                   | 305/2499 [37:39<7:29:50, 12.30s/it]
                                                                                                                                                           
{'loss': 0.5736, 'grad_norm': 0.14988955855369568, 'learning_rate': 0.00019319347828701563, 'ppl': 1.7746, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4556.19, 'total_tokens': 13855721, 'epoch': 0.37}
-
 12%|█████████████▊                                                                                                   | 305/2499 [37:39<7:29:50, 12.30s/it]
 12%|█████████████▊                                                                                                   | 306/2499 [37:45<6:24:12, 10.51s/it]
                                                                                                                                                           
{'loss': 0.5878, 'grad_norm': 0.16112789511680603, 'learning_rate': 0.0001931476337578762, 'ppl': 1.8, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4654.1, 'total_tokens': 13885195, 'epoch': 0.37}
-
 12%|█████████████▊                                                                                                   | 306/2499 [37:45<6:24:12, 10.51s/it]
 12%|█████████████▉                                                                                                   | 307/2499 [37:52<5:38:13,  9.26s/it]
                                                                                                                                                           
{'loss': 0.5895, 'grad_norm': 0.15954813361167908, 'learning_rate': 0.0001931016408328985, 'ppl': 1.8031, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4589.73, 'total_tokens': 13914238, 'epoch': 0.37}
-
 12%|█████████████▉                                                                                                   | 307/2499 [37:52<5:38:13,  9.26s/it]
 12%|█████████████▉                                                                                                   | 308/2499 [37:58<5:05:36,  8.37s/it]
                                                                                                                                                           
{'loss': 0.5728, 'grad_norm': 0.1708773374557495, 'learning_rate': 0.000193055499585355, 'ppl': 1.7732, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4591.58, 'total_tokens': 13943121, 'epoch': 0.37}
-
 12%|█████████████▉                                                                                                   | 308/2499 [37:58<5:05:36,  8.37s/it]
 12%|█████████████▉                                                                                                   | 309/2499 [38:04<4:42:23,  7.74s/it]
                                                                                                                                                           
{'loss': 0.5704, 'grad_norm': 0.15702755749225616, 'learning_rate': 0.00019300921008875453, 'ppl': 1.769, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4347.24, 'total_tokens': 13970336, 'epoch': 0.37}
-
 12%|█████████████▉                                                                                                   | 309/2499 [38:04<4:42:23,  7.74s/it]
 12%|██████████████                                                                                                   | 310/2499 [38:11<4:26:25,  7.30s/it]
                                                                                                                                                           
{'loss': 0.5649, 'grad_norm': 0.14557598531246185, 'learning_rate': 0.00019296277241684194, 'ppl': 1.7593, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4666.71, 'total_tokens': 13999675, 'epoch': 0.37}
-
 12%|███████████��██                                                                                                   | 310/2499 [38:11<4:26:25,  7.30s/it]
 12%|██████████████                                                                                                   | 311/2499 [38:17<4:15:09,  7.00s/it]
                                                                                                                                                           
{'loss': 0.5825, 'grad_norm': 0.17242364585399628, 'learning_rate': 0.00019291618664359837, 'ppl': 1.7905, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4605.65, 'total_tokens': 14028606, 'epoch': 0.37}
-
 12%|██████████████                                                                                                   | 311/2499 [38:17<4:15:09,  7.00s/it]
 12%|██████████████                                                                                                   | 312/2499 [38:23<4:06:59,  6.78s/it]
                                                                                                                                                           
{'loss': 0.6062, 'grad_norm': 0.17027081549167633, 'learning_rate': 0.00019286945284324074, 'ppl': 1.8335, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4397.97, 'total_tokens': 14056129, 'epoch': 0.37}
-
 12%|██████████████                                                                                                   | 312/2499 [38:23<4:06:59,  6.78s/it]
 13%|██████████████▏                                                                                                  | 313/2499 [38:29<4:01:45,  6.64s/it]
                                                                                                                                                           
{'loss': 0.5704, 'grad_norm': 0.15852947533130646, 'learning_rate': 0.00019282257109022182, 'ppl': 1.769, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4571.76, 'total_tokens': 14084955, 'epoch': 0.38}
-
 13%|██████████████▏                                                                                                  | 313/2499 [38:29<4:01:45,  6.64s/it]
 13%|██████████████▏                                                                                                  | 314/2499 [38:36<3:58:25,  6.55s/it]
                                                                                                                                                           
{'loss': 0.5697, 'grad_norm': 0.15593717992305756, 'learning_rate': 0.00019277554145923014, 'ppl': 1.7677, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4672.64, 'total_tokens': 14114562, 'epoch': 0.38}
-
 13%|██████████████▏                                                                                                  | 314/2499 [38:36<3:58:25,  6.55s/it]
 13%|██████████████▏                                                                                                  | 315/2499 [38:42<3:55:28,  6.47s/it]
                                                                                                                                                           
{'loss': 0.6057, 'grad_norm': 0.1503870040178299, 'learning_rate': 0.0001927283640251898, 'ppl': 1.8325, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4547.99, 'total_tokens': 14143136, 'epoch': 0.38}
-
 13%|██████████████▏                                                                                                  | 315/2499 [38:42<3:55:28,  6.47s/it]
 13%|██████████████▎                                                                                                  | 316/2499 [38:48<3:53:17,  6.41s/it]
                                                                                                                                                           
{'loss': 0.608, 'grad_norm': 0.1603906899690628, 'learning_rate': 0.00019268103886326031, 'ppl': 1.8368, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4632.34, 'total_tokens': 14172210, 'epoch': 0.38}
-
 13%|██████████████▎                                                                                                  | 316/2499 [38:48<3:53:17,  6.41s/it]
 13%|█████████���████▎                                                                                                  | 317/2499 [38:55<3:51:42,  6.37s/it]
                                                                                                                                                           
{'loss': 0.5695, 'grad_norm': 0.159032940864563, 'learning_rate': 0.00019263356604883663, 'ppl': 1.7674, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4531.06, 'total_tokens': 14200637, 'epoch': 0.38}
-
 13%|██████████████▎                                                                                                  | 317/2499 [38:55<3:51:42,  6.37s/it]
 13%|██████████████▍                                                                                                  | 318/2499 [39:01<3:50:39,  6.35s/it]
                                                                                                                                                           
{'loss': 0.5604, 'grad_norm': 0.14921818673610687, 'learning_rate': 0.00019258594565754885, 'ppl': 1.7514, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4684.39, 'total_tokens': 14230071, 'epoch': 0.38}
-
 13%|██████████████▍                                                                                                  | 318/2499 [39:01<3:50:39,  6.35s/it]
 13%|██████████████▍                                                                                                  | 319/2499 [39:07<3:49:35,  6.32s/it]
                                                                                                                                                           
{'loss': 0.5661, 'grad_norm': 0.1620176136493683, 'learning_rate': 0.00019253817776526222, 'ppl': 1.7614, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4380.21, 'total_tokens': 14257470, 'epoch': 0.38}
-
 13%|██████████████▍                                                                                                  | 319/2499 [39:07<3:49:35,  6.32s/it]
 13%|██████████████▍                                                                                                  | 320/2499 [39:13<3:49:31,  6.32s/it]
                                                                                                                                                           
{'loss': 0.5905, 'grad_norm': 0.18213319778442383, 'learning_rate': 0.00019249026244807702, 'ppl': 1.8049, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4426.57, 'total_tokens': 14285439, 'epoch': 0.38}
-
 13%|██████████████▍                                                                                                  | 320/2499 [39:13<3:49:31,  6.32s/it]
 13%|██████████████▌                                                                                                  | 321/2499 [39:20<3:49:34,  6.32s/it]
                                                                                                                                                           
{'loss': 0.6033, 'grad_norm': 0.16348907351493835, 'learning_rate': 0.0001924421997823283, 'ppl': 1.8281, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4607.25, 'total_tokens': 14314609, 'epoch': 0.39}
-
 13%|██████████████▌                                                                                                  | 321/2499 [39:20<3:49:34,  6.32s/it]
 13%|██████████████▌                                                                                                  | 322/2499 [39:26<3:48:57,  6.31s/it]
                                                                                                                                                           
{'loss': 0.5923, 'grad_norm': 0.17127369344234467, 'learning_rate': 0.00019239398984458587, 'ppl': 1.8081, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4518.62, 'total_tokens': 14342958, 'epoch': 0.39}
-
 13%|██████████████▌                                                                                                  | 322/2499 [39:26<3:48:57,  6.31s/it]
 13%|██████████████▌                                                                                                  | 323/2499 [39:32<3:48:21,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6389, 'grad_norm': 0.16117724776268005, 'learning_rate': 0.00019234563271165427, 'ppl': 1.8944, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4450.13, 'total_tokens': 14370831, 'epoch': 0.39}
-
 13%|██████████████▌                                                                                                  | 323/2499 [39:32<3:48:21,  6.30s/it]
 13%|██████████████▋                                                                                                  | 324/2499 [39:39<3:47:59,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5963, 'grad_norm': 0.1653064787387848, 'learning_rate': 0.0001922971284605724, 'ppl': 1.8154, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4532.05, 'total_tokens': 14399248, 'epoch': 0.39}
-
 13%|██████████████▋                                                                                                  | 324/2499 [39:39<3:47:59,  6.29s/it]
 13%|██████████████▋                                                                                                  | 325/2499 [39:45<3:47:47,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6196, 'grad_norm': 0.1635480523109436, 'learning_rate': 0.00019224847716861368, 'ppl': 1.8582, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4648.34, 'total_tokens': 14428429, 'epoch': 0.39}
-
 13%|██████████████▋                                                                                                  | 325/2499 [39:45<3:47:47,  6.29s/it]
 13%|██████████████▋                                                                                                  | 326/2499 [39:51<3:47:35,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6349, 'grad_norm': 0.16426773369312286, 'learning_rate': 0.00019219967891328564, 'ppl': 1.8868, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4478.02, 'total_tokens': 14456533, 'epoch': 0.39}
-
 13%|██████████████▋                                                                                                  | 326/2499 [39:51<3:47:35,  6.28s/it]
 13%|██████████████▊                                                                                                  | 327/2499 [39:57<3:47:52,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5799, 'grad_norm': 0.16964776813983917, 'learning_rate': 0.00019215073377233002, 'ppl': 1.7859, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4515.9, 'total_tokens': 14485063, 'epoch': 0.39}
-
 13%|██████████████▊                                                                                                  | 327/2499 [39:57<3:47:52,  6.29s/it]
 13%|██████████████▊                                                                                                  | 328/2499 [40:04<3:47:44,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5549, 'grad_norm': 0.16128554940223694, 'learning_rate': 0.00019210164182372256, 'ppl': 1.7418, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4576.6, 'total_tokens': 14513839, 'epoch': 0.39}
-
 13%|██████████████▊                                                                                                  | 328/2499 [40:04<3:47:44,  6.29s/it]
 13%|██████████████▉                                                                                                  | 329/2499 [40:10<3:47:23,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5898, 'grad_norm': 0.17319965362548828, 'learning_rate': 0.0001920524031456729, 'ppl': 1.8036, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4489.45, 'total_tokens': 14541984, 'epoch': 0.39}
-
 13%|██████████████▉                                                                                                  | 329/2499 [40:10<3:47:23,  6.29s/it]
 13%|██████████████▉                                                                                                  | 330/2499 [40:16<3:47:13,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5479, 'grad_norm': 0.15397316217422485, 'learning_rate': 0.00019200301781662437, 'ppl': 1.7296, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4650.57, 'total_tokens': 14571189, 'epoch': 0.4}
-
 13%|██████████████▉                                                                                                  | 330/2499 [40:16<3:47:13,  6.29s/it]
 13%|██████████████▉                                                                                                  | 331/2499 [40:23<3:47:08,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5794, 'grad_norm': 0.16392205655574799, 'learning_rate': 0.00019195348591525402, 'ppl': 1.785, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4661.49, 'total_tokens': 14600485, 'epoch': 0.4}
-
 13%|██████████████▉                                                                                                  | 331/2499 [40:23<3:47:08,  6.29s/it]
 13%|███████████████                                                                                                  | 332/2499 [40:29<3:46:50,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5789, 'grad_norm': 0.15939205884933472, 'learning_rate': 0.00019190380752047233, 'ppl': 1.7841, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4511.16, 'total_tokens': 14628748, 'epoch': 0.4}
-
 13%|███████████████                                                                                                  | 332/2499 [40:29<3:46:50,  6.28s/it]
 13%|███████████████                                                                                                  | 333/2499 [40:35<3:46:42,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6088, 'grad_norm': 0.13913695514202118, 'learning_rate': 0.00019185398271142327, 'ppl': 1.8382, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4650.63, 'total_tokens': 14657935, 'epoch': 0.4}
-
 13%|███████████████                                                                                                  | 333/2499 [40:35<3:46:42,  6.28s/it]
 13%|███████████████                                                                                                  | 334/2499 [40:41<3:47:09,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6075, 'grad_norm': 0.16204483807086945, 'learning_rate': 0.00019180401156748396, 'ppl': 1.8358, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4602.41, 'total_tokens': 14687062, 'epoch': 0.4}
-
 13%|███████████████                                                                                                  | 334/2499 [40:41<3:47:09,  6.30s/it]
 13%|███████████████▏                                                                                                 | 335/2499 [40:48<3:47:07,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5733, 'grad_norm': 0.1686972677707672, 'learning_rate': 0.00019175389416826462, 'ppl': 1.7741, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4672.53, 'total_tokens': 14716493, 'epoch': 0.4}
-
 13%|███████████████▏                                                                                                 | 335/2499 [40:48<3:47:07,  6.30s/it]
 13%|███████████████▏                                                                                                 | 336/2499 [40:54<3:47:04,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5534, 'grad_norm': 0.1636599749326706, 'learning_rate': 0.00019170363059360863, 'ppl': 1.7392, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4694.41, 'total_tokens': 14746072, 'epoch': 0.4}
-
 13%|███████████████▏                                                                                                 | 336/2499 [40:54<3:47:04,  6.30s/it]
 13%|███████████████▏                                                                                                 | 337/2499 [41:00<3:46:38,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5535, 'grad_norm': 0.14456257224082947, 'learning_rate': 0.00019165322092359213, 'ppl': 1.7393, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4609.38, 'total_tokens': 14774955, 'epoch': 0.4}
-
 13%|███████████████▏                                                                                                 | 337/2499 [41:00<3:46:38,  6.29s/it]
 14%|███████████████▎                                                                                                 | 338/2499 [41:07<3:46:34,  6.29s/it]
                                                                                                                                                           
{'loss': 0.596, 'grad_norm': 0.160542830824852, 'learning_rate': 0.00019160266523852402, 'ppl': 1.8148, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4724.33, 'total_tokens': 14804675, 'epoch': 0.41}
-
 14%|███████████████▎                                                                                                 | 338/2499 [41:07<3:46:34,  6.29s/it]
 14%|███████████████▎                                                                                                 | 339/2499 [41:13<3:46:08,  6.28s/it]
                                                                                                                                                           
{'loss': 0.543, 'grad_norm': 0.15767574310302734, 'learning_rate': 0.00019155196361894582, 'ppl': 1.7212, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4489.42, 'total_tokens': 14832767, 'epoch': 0.41}
-
 14%|███████████████▎                                                                                                 | 339/2499 [41:13<3:46:08,  6.28s/it]
 14%|███████████████▎                                                                                                 | 340/2499 [41:19<3:45:58,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5923, 'grad_norm': 0.16006726026535034, 'learning_rate': 0.00019150111614563161, 'ppl': 1.8081, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4385.16, 'total_tokens': 14860277, 'epoch': 0.41}
-
 14%|███████████████▎                                                                                                 | 340/2499 [41:19<3:45:58,  6.28s/it]
 14%|███████████████▍                                                                                                 | 341/2499 [41:25<3:46:15,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6217, 'grad_norm': 0.15745171904563904, 'learning_rate': 0.00019145012289958772, 'ppl': 1.8621, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4424.68, 'total_tokens': 14888207, 'epoch': 0.41}
-
 14%|██████████████���▍                                                                                                 | 341/2499 [41:25<3:46:15,  6.29s/it]
 14%|███████████████▍                                                                                                 | 342/2499 [41:32<3:45:48,  6.28s/it]
                                                                                                                                                           
{'loss': 0.63, 'grad_norm': 0.18214435875415802, 'learning_rate': 0.0001913989839620528, 'ppl': 1.8776, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4404.68, 'total_tokens': 14915755, 'epoch': 0.41}
-
 14%|███████████████▍                                                                                                 | 342/2499 [41:32<3:45:48,  6.28s/it]
 14%|███████████████▌                                                                                                 | 343/2499 [41:38<3:45:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5884, 'grad_norm': 0.17168237268924713, 'learning_rate': 0.00019134769941449756, 'ppl': 1.8011, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4431.31, 'total_tokens': 14943470, 'epoch': 0.41}
-
 14%|███████████████▌                                                                                                 | 343/2499 [41:38<3:45:26,  6.27s/it]
 14%|███████████████▌                                                                                                 | 344/2499 [41:44<3:45:20,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6061, 'grad_norm': 0.17049424350261688, 'learning_rate': 0.00019129626933862473, 'ppl': 1.8333, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4507.98, 'total_tokens': 14971748, 'epoch': 0.41}
-
 14%|███████████████▌                                                                                                 | 344/2499 [41:44<3:45:20,  6.27s/it]
 14%|███████████████▌                                                                                                 | 345/2499 [41:51<3:45:15,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5698, 'grad_norm': 0.14557531476020813, 'learning_rate': 0.00019124469381636886, 'ppl': 1.7679, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4549.13, 'total_tokens': 15000282, 'epoch': 0.41}
-
 14%|███████████████▌                                                                                                 | 345/2499 [41:51<3:45:15,  6.27s/it]
 14%|███████████████▋                                                                                                 | 346/2499 [41:57<3:45:37,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5226, 'grad_norm': 0.14654414355754852, 'learning_rate': 0.0001911929729298962, 'ppl': 1.6864, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4599.13, 'total_tokens': 15029331, 'epoch': 0.42}
-
 14%|███████████████▋                                                                                                 | 346/2499 [41:57<3:45:37,  6.29s/it]
 14%|███████████████▋                                                                                                 | 347/2499 [42:03<3:46:10,  6.31s/it]
                                                                                                                                                           
{'loss': 0.5269, 'grad_norm': 0.14888443052768707, 'learning_rate': 0.00019114110676160455, 'ppl': 1.6937, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4587.79, 'total_tokens': 15058447, 'epoch': 0.42}
-
 14%|███████████████▋                                                                                                 | 347/2499 [42:03<3:46:10,  6.31s/it]
 14%|███████████████▋                                                                                                 | 348/2499 [42:10<3:46:38,  6.32s/it]
                                                                                                                                                           
{'loss': 0.5954, 'grad_norm': 0.17817890644073486, 'learning_rate': 0.0001910890953941233, 'ppl': 1.8138, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4546.01, 'total_tokens': 15087338, 'epoch': 0.42}
-
 14%|███████████████▋                                                                                                 | 348/2499 [42:10<3:46:38,  6.32s/it]
 14%|███████████████▊                                                                                                 | 349/2499 [42:16<3:46:01,  6.31s/it]
                                                                                                                                                           
{'loss': 0.608, 'grad_norm': 0.16011007130146027, 'learning_rate': 0.00019103693891031295, 'ppl': 1.8368, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4504.66, 'total_tokens': 15115585, 'epoch': 0.42}
-
 14%|███████████████▊                                                                                                 | 349/2499 [42:16<3:46:01,  6.31s/it]
 14%|███████████████▊                                                                                                 | 350/2499 [42:22<3:45:24,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5941, 'grad_norm': 0.1592024266719818, 'learning_rate': 0.00019098463739326542, 'ppl': 1.8114, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4487.29, 'total_tokens': 15143669, 'epoch': 0.42}
-
 14%|███████████████▊                                                                                                 | 350/2499 [42:22<3:45:24,  6.29s/it]
 14%|███████████████▊                                                                                                 | 351/2499 [42:28<3:45:02,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6043, 'grad_norm': 0.15884292125701904, 'learning_rate': 0.00019093219092630352, 'ppl': 1.83, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4541.37, 'total_tokens': 15172129, 'epoch': 0.42}
-
 14%|███████████████▊                                                                                                 | 351/2499 [42:28<3:45:02,  6.29s/it]
 14%|███████████████▉                                                                                                 | 352/2499 [42:35<3:44:53,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6089, 'grad_norm': 0.162332221865654, 'learning_rate': 0.00019087959959298103, 'ppl': 1.8384, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4552.12, 'total_tokens': 15200710, 'epoch': 0.42}
-
 14%|███████████████▉                                                                                                 | 352/2499 [42:35<3:44:53,  6.28s/it]
 14%|███████████████▉                                                                                                 | 353/2499 [42:41<3:44:46,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5484, 'grad_norm': 0.14525768160820007, 'learning_rate': 0.00019082686347708254, 'ppl': 1.7305, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4662.96, 'total_tokens': 15230000, 'epoch': 0.42}
-
 14%|███████████████▉                                                                                                 | 353/2499 [42:41<3:44:46,  6.28s/it]
 14%|████████████████                                                                                                 | 354/2499 [42:47<3:44:58,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6355, 'grad_norm': 0.14745378494262695, 'learning_rate': 0.0001907739826626233, 'ppl': 1.888, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4550.69, 'total_tokens': 15258721, 'epoch': 0.42}
-
 14%|████████████████                                                                                                 | 354/2499 [42:47<3:44:58,  6.29s/it]
 14%|████████████████                                                                                                 | 355/2499 [42:54<3:44:56,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5874, 'grad_norm': 0.1704423427581787, 'learning_rate': 0.00019072095723384903, 'ppl': 1.7993, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4349.01, 'total_tokens': 15286100, 'epoch': 0.43}
-
 14%|████████████████                                                                                                 | 355/2499 [42:54<3:44:56,  6.30s/it]
 14%|████████████████                                                                                                 | 356/2499 [43:00<3:44:38,  6.29s/it]
                                                                                                                                                           
{'loss': 0.569, 'grad_norm': 0.15435314178466797, 'learning_rate': 0.00019066778727523586, 'ppl': 1.7665, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4527.29, 'total_tokens': 15314498, 'epoch': 0.43}
-
 14%|████████████████                                                                                                 | 356/2499 [43:00<3:44:38,  6.29s/it]
 14%|████████████████▏                                                                                                | 357/2499 [43:06<3:44:36,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5769, 'grad_norm': 0.15472789108753204, 'learning_rate': 0.00019061447287149026, 'ppl': 1.7805, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4474.29, 'total_tokens': 15342655, 'epoch': 0.43}
-
 14%|████████████████▏                                                                                                | 357/2499 [43:06<3:44:36,  6.29s/it]
 14%|████████████████▏                                                                                                | 358/2499 [43:12<3:44:16,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5835, 'grad_norm': 0.160075843334198, 'learning_rate': 0.0001905610141075487, 'ppl': 1.7923, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4471.53, 'total_tokens': 15370684, 'epoch': 0.43}
-
 14%|████████████████▏                                                                                                | 358/2499 [43:12<3:44:16,  6.29s/it]
 14%|████████████████▏                                                                                                | 359/2499 [43:19<3:44:07,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6181, 'grad_norm': 0.18926472961902618, 'learning_rate': 0.00019050741106857767, 'ppl': 1.8554, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4563.78, 'total_tokens': 15399336, 'epoch': 0.43}
-
 14%|████████████████▏                                                                                                | 359/2499 [43:19<3:44:07,  6.28s/it]
 14%|████████████████▎                                                                                                | 360/2499 [43:25<3:43:51,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5702, 'grad_norm': 0.14512301981449127, 'learning_rate': 0.00019045366383997354, 'ppl': 1.7686, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4667.64, 'total_tokens': 15428588, 'epoch': 0.43}
-
 14%|████████████████▎                                                                                                | 360/2499 [43:25<3:43:51,  6.28s/it]
 14%|████████████████▎                                                                                                | 361/2499 [43:31<3:44:06,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5866, 'grad_norm': 0.1660764068365097, 'learning_rate': 0.0001903997725073624, 'ppl': 1.7979, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4457.92, 'total_tokens': 15456714, 'epoch': 0.43}
-
 14%|████████████████▎                                                                                                | 361/2499 [43:31<3:44:06,  6.29s/it]
 14%|████████████████▎                                                                                                | 362/2499 [43:38<3:44:08,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5781, 'grad_norm': 0.15140269696712494, 'learning_rate': 0.0001903457371565998, 'ppl': 1.7826, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4605.76, 'total_tokens': 15485721, 'epoch': 0.43}
-
 14%|████████████████▎                                                                                                | 362/2499 [43:38<3:44:08,  6.29s/it]
 15%|████████████████▍                                                                                                | 363/2499 [43:44<3:43:58,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5577, 'grad_norm': 0.15968385338783264, 'learning_rate': 0.0001902915578737709, 'ppl': 1.7467, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4660.53, 'total_tokens': 15515013, 'epoch': 0.44}
-
 15%|████████████████▍                                                                                                | 363/2499 [43:44<3:43:58,  6.29s/it]
 15%|████████████████▍                                                                                                | 364/2499 [43:50<3:43:44,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5382, 'grad_norm': 0.14655247330665588, 'learning_rate': 0.00019023723474519003, 'ppl': 1.7129, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4668.13, 'total_tokens': 15544312, 'epoch': 0.44}
-
 15%|████████████████▍                                                                                                | 364/2499 [43:50<3:43:44,  6.29s/it]
 15%|████████████████▌                                                                                                | 365/2499 [43:56<3:43:26,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5903, 'grad_norm': 0.16471044719219208, 'learning_rate': 0.00019018276785740077, 'ppl': 1.8045, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4603.6, 'total_tokens': 15573163, 'epoch': 0.44}
-
 15%|████████████████▌                                                                                                | 365/2499 [43:56<3:43:26,  6.28s/it]
 15%|████████████████▌                                                                                                | 366/2499 [44:03<3:43:08,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6285, 'grad_norm': 0.16121886670589447, 'learning_rate': 0.00019012815729717568, 'ppl': 1.8748, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4528.92, 'total_tokens': 15601521, 'epoch': 0.44}
-
 15%|████████████████▌                                                                                                | 366/2499 [44:03<3:43:08,  6.28s/it]
 15%|████████████████▌                                                                                                | 367/2499 [44:09<3:42:38,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5723, 'grad_norm': 0.15570220351219177, 'learning_rate': 0.0001900734031515162, 'ppl': 1.7723, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4270.92, 'total_tokens': 15628162, 'epoch': 0.44}
-
 15%|████████████████▌                                                                                                | 367/2499 [44:09<3:42:38,  6.27s/it]
 15%|████████████████▋                                                                                                | 368/2499 [44:15<3:42:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5747, 'grad_norm': 0.1494811475276947, 'learning_rate': 0.00019001850550765258, 'ppl': 1.7766, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4639.58, 'total_tokens': 15657404, 'epoch': 0.44}
-
 15%|████████████████▋                                                                                                | 368/2499 [44:15<3:42:57,  6.28s/it]
 15%|████████████████▋                                                                                                | 369/2499 [44:21<3:42:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5729, 'grad_norm': 0.15576674044132233, 'learning_rate': 0.00018996346445304357, 'ppl': 1.7734, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4490.94, 'total_tokens': 15685549, 'epoch': 0.44}
-
 15%|████████████████▋                                                                                                | 369/2499 [44:21<3:42:47,  6.28s/it]
 15%|████████████████▋                                                                                                | 370/2499 [44:28<3:42:24,  6.27s/it]
                                                                                                                                                           
{'loss': 0.568, 'grad_norm': 0.16379213333129883, 'learning_rate': 0.00018990828007537646, 'ppl': 1.7647, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4541.16, 'total_tokens': 15713920, 'epoch': 0.44}
-
 15%|████████████████▋                                                                                                | 370/2499 [44:28<3:42:24,  6.27s/it]
 15%|████████████████▊                                                                                                | 371/2499 [44:34<3:42:25,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5741, 'grad_norm': 0.15409880876541138, 'learning_rate': 0.00018985295246256694, 'ppl': 1.7755, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4750.89, 'total_tokens': 15743743, 'epoch': 0.45}
-
 15%|████████████████▊                                                                                                | 371/2499 [44:34<3:42:25,  6.27s/it]
 15%|████████████████▊                                                                                                | 372/2499 [44:40<3:42:16,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6448, 'grad_norm': 0.16626456379890442, 'learning_rate': 0.0001897974817027588, 'ppl': 1.9056, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4482.9, 'total_tokens': 15771824, 'epoch': 0.45}
-
 15%|████████████████▊                                                                                                | 372/2499 [44:40<3:42:16,  6.27s/it]
 15%|████████████████▊                                                                                                | 373/2499 [44:47<3:42:05,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6312, 'grad_norm': 0.17316874861717224, 'learning_rate': 0.00018974186788432384, 'ppl': 1.8799, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4500.96, 'total_tokens': 15800000, 'epoch': 0.45}
-
 15%|████████████████▊                                                                                                | 373/2499 [44:47<3:42:05,  6.27s/it]
 15%|████████████████▉                                                                                                | 374/2499 [44:53<3:42:08,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5917, 'grad_norm': 0.16619497537612915, 'learning_rate': 0.00018968611109586192, 'ppl': 1.8071, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4609.35, 'total_tokens': 15828947, 'epoch': 0.45}
-
 15%|████████████████▉                                                                                                | 374/2499 [44:53<3:42:08,  6.27s/it]
 15%|████████████████▉                                                                                                | 375/2499 [44:59<3:42:18,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6261, 'grad_norm': 0.15712764859199524, 'learning_rate': 0.00018963021142620053, 'ppl': 1.8703, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4563.22, 'total_tokens': 15857673, 'epoch': 0.45}
-
 15%|████████████████▉                                                                                                | 375/2499 [44:59<3:42:18,  6.28s/it]
 15%|█████████████████                                                                                                | 376/2499 [45:05<3:42:40,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6129, 'grad_norm': 0.16614432632923126, 'learning_rate': 0.00018957416896439483, 'ppl': 1.8458, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4639.79, 'total_tokens': 15887003, 'epoch': 0.45}
-
 15%|█████████████████                                                                                                | 376/2499 [45:05<3:42:40,  6.29s/it]
 15%|█████████████████                                                                                                | 377/2499 [45:12<3:42:52,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6212, 'grad_norm': 0.15233156085014343, 'learning_rate': 0.0001895179837997275, 'ppl': 1.8612, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4427.06, 'total_tokens': 15914974, 'epoch': 0.45}
-
 15%|█████████████████                                                                                                | 377/2499 [45:12<3:42:52,  6.30s/it]
 15%|█████████████████                                                                                                | 378/2499 [45:18<3:42:39,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6029, 'grad_norm': 0.16825121641159058, 'learning_rate': 0.00018946165602170856, 'ppl': 1.8274, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4614.83, 'total_tokens': 15943986, 'epoch': 0.45}
-
 15%|█████████████████                                                                                                | 378/2499 [45:18<3:42:39,  6.30s/it]
 15%|█████████████████▏                                                                                               | 379/2499 [45:24<3:42:26,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5672, 'grad_norm': 0.1466662883758545, 'learning_rate': 0.00018940518572007516, 'ppl': 1.7633, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4758.72, 'total_tokens': 15973898, 'epoch': 0.45}
-
 15%|█████████████████▏                                                                                               | 379/2499 [45:24<3:42:26,  6.30s/it]
 15%|█████████████████▏                                                                                               | 380/2499 [45:31<3:41:59,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6395, 'grad_norm': 0.16290511190891266, 'learning_rate': 0.0001893485729847916, 'ppl': 1.8955, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4498.14, 'total_tokens': 16002060, 'epoch': 0.46}
-
 15%|█████████████████▏                                                                                               | 380/2499 [45:31<3:41:59,  6.29s/it]
 15%|█████████████████▏                                                                                               | 381/2499 [45:37<3:41:35,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5917, 'grad_norm': 0.1727246642112732, 'learning_rate': 0.00018929181790604903, 'ppl': 1.8071, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4368.8, 'total_tokens': 16029392, 'epoch': 0.46}
-
 15%|█████████████████▏                                                                                               | 381/2499 [45:37<3:41:35,  6.28s/it]
 15%|█████████████████▎                                                                                               | 382/2499 [45:43<3:41:27,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5887, 'grad_norm': 0.1509438455104828, 'learning_rate': 0.0001892349205742654, 'ppl': 1.8016, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4618.89, 'total_tokens': 16058358, 'epoch': 0.46}
-
 15%|█████████████████▎                                                                                               | 382/2499 [45:43<3:41:27,  6.28s/it]
 15%|█████████████████▎                                                                                               | 383/2499 [45:49<3:41:24,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5796, 'grad_norm': 0.15395300090312958, 'learning_rate': 0.00018917788108008528, 'ppl': 1.7853, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4309.92, 'total_tokens': 16085426, 'epoch': 0.46}
-
 15%|█████████████████▎                                                                                               | 383/2499 [45:49<3:41:24,  6.28s/it]
 15%|█████████████████▎                                                                                               | 384/2499 [45:56<3:41:13,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5248, 'grad_norm': 0.13965602219104767, 'learning_rate': 0.00018912069951437972, 'ppl': 1.6901, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4440.01, 'total_tokens': 16113248, 'epoch': 0.46}
-
 15%|█████████████████▎                                                                                               | 384/2499 [45:56<3:41:13,  6.28s/it]
 15%|█████████████████▍                                                                                               | 385/2499 [46:02<3:40:54,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6001, 'grad_norm': 0.1578795462846756, 'learning_rate': 0.00018906337596824612, 'ppl': 1.8223, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4435.44, 'total_tokens': 16140981, 'epoch': 0.46}
-
 15%|█████████████████▍                                                                                               | 385/2499 [46:02<3:40:54,  6.27s/it]
 15%|█████████████████▍                                                                                               | 386/2499 [46:08<3:40:36,  6.26s/it]
                                                                                                                                                           
{'loss': 0.4905, 'grad_norm': 0.14027678966522217, 'learning_rate': 0.00018900591053300805, 'ppl': 1.6331, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4402.48, 'total_tokens': 16168495, 'epoch': 0.46}
-
 15%|█████████████████▍                                                                                               | 386/2499 [46:08<3:40:36,  6.26s/it]
 15%|█████████████████▍                                                                                               | 387/2499 [46:14<3:40:28,  6.26s/it]
                                                                                                                                                           
{'loss': 0.6212, 'grad_norm': 0.16160684823989868, 'learning_rate': 0.00018894830330021516, 'ppl': 1.8612, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4488.09, 'total_tokens': 16196583, 'epoch': 0.46}
-
 15%|█████████████████▍                                                                                               | 387/2499 [46:14<3:40:28,  6.26s/it]
 16%|█████████████████▌                                                                                               | 388/2499 [46:21<3:40:20,  6.26s/it]
                                                                                                                                                           
{'loss': 0.6166, 'grad_norm': 0.1768082231283188, 'learning_rate': 0.00018889055436164295, 'ppl': 1.8526, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4527.83, 'total_tokens': 16224926, 'epoch': 0.47}
-
 16%|█████████████████▌                                                                                               | 388/2499 [46:21<3:40:20,  6.26s/it]
 16%|█████████████████▌                                                                                               | 389/2499 [46:27<3:40:30,  6.27s/it]
                                                                                                                                                           
{'loss': 0.581, 'grad_norm': 0.16775575280189514, 'learning_rate': 0.00018883266380929271, 'ppl': 1.7878, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4346.76, 'total_tokens': 16252245, 'epoch': 0.47}
-
 16%|█████████████████▌                                                                                               | 389/2499 [46:27<3:40:30,  6.27s/it]
 16%|█████████████████▋                                                                                               | 390/2499 [46:33<3:40:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5907, 'grad_norm': 0.1857319474220276, 'learning_rate': 0.00018877463173539138, 'ppl': 1.8053, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4364.59, 'total_tokens': 16279755, 'epoch': 0.47}
-
 16%|█████████████████▋                                                                                               | 390/2499 [46:33<3:40:47,  6.28s/it]
 16%|█████████████████▋                                                                                               | 391/2499 [46:40<3:40:38,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5743, 'grad_norm': 0.19177928566932678, 'learning_rate': 0.00018871645823239128, 'ppl': 1.7759, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4530.49, 'total_tokens': 16308173, 'epoch': 0.47}
-
 16%|█████████████████▋                                                                                               | 391/2499 [46:40<3:40:38,  6.28s/it]
 16%|█████████████████▋                                                                                               | 392/2499 [46:46<3:40:19,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6016, 'grad_norm': 0.16839760541915894, 'learning_rate': 0.0001886581433929701, 'ppl': 1.825, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4453.3, 'total_tokens': 16336041, 'epoch': 0.47}
-
 16%|█████████████████▋                                                                                               | 392/2499 [46:46<3:40:19,  6.27s/it]
 16%|█████████████████▊                                                                                               | 393/2499 [46:52<3:40:09,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5647, 'grad_norm': 0.1491001546382904, 'learning_rate': 0.00018859968731003063, 'ppl': 1.7589, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4608.54, 'total_tokens': 16364916, 'epoch': 0.47}
-
 16%|█████████████████▊                                                                                               | 393/2499 [46:52<3:40:09,  6.27s/it]
 16%|█████████████████▊                                                                                               | 394/2499 [46:58<3:39:48,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5319, 'grad_norm': 0.14710378646850586, 'learning_rate': 0.0001885410900767008, 'ppl': 1.7022, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4340.71, 'total_tokens': 16392033, 'epoch': 0.47}
-
 16%|█████████████████▊                                                                                               | 394/2499 [46:58<3:39:48,  6.27s/it]
 16%|█████████████████▊                                                                                               | 395/2499 [47:05<3:39:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5727, 'grad_norm': 0.16849607229232788, 'learning_rate': 0.00018848235178633326, 'ppl': 1.773, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4629.61, 'total_tokens': 16421090, 'epoch': 0.47}
-
 16%|█████████████████▊                                                                                               | 395/2499 [47:05<3:39:50,  6.27s/it]
 16%|█████████████████▉                                                                                               | 396/2499 [47:11<3:40:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5808, 'grad_norm': 0.16942906379699707, 'learning_rate': 0.00018842347253250554, 'ppl': 1.7875, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4589.99, 'total_tokens': 16449981, 'epoch': 0.48}
-
 16%|█████████████████▉                                                                                               | 396/2499 [47:11<3:40:01,  6.28s/it]
 16%|█████████████████▉                                                                                               | 397/2499 [47:17<3:40:09,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5571, 'grad_norm': 0.1529962718486786, 'learning_rate': 0.00018836445240901956, 'ppl': 1.7456, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4469.78, 'total_tokens': 16478120, 'epoch': 0.48}
-
 16%|█████████████████▉                                                                                               | 397/2499 [47:17<3:40:09,  6.28s/it]
 16%|█████████████████▉                                                                                               | 398/2499 [47:23<3:39:44,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6531, 'grad_norm': 0.16236943006515503, 'learning_rate': 0.00018830529150990185, 'ppl': 1.9215, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4429.27, 'total_tokens': 16505806, 'epoch': 0.48}
-
 16%|█████████████████▉                                                                                               | 398/2499 [47:23<3:39:44,  6.28s/it]
 16%|██████████████████                                                                                               | 399/2499 [47:30<3:39:33,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5702, 'grad_norm': 0.15536250174045563, 'learning_rate': 0.00018824598992940312, 'ppl': 1.7686, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4621.37, 'total_tokens': 16534759, 'epoch': 0.48}
-
 16%|██████████████████                                                                                               | 399/2499 [47:30<3:39:33,  6.27s/it]
 16%|██████████████████                                                                                               | 400/2499 [47:36<3:39:25,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5201, 'grad_norm': 0.15856070816516876, 'learning_rate': 0.00018818654776199816, 'ppl': 1.6822, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4552.77, 'total_tokens': 16563296, 'epoch': 0.48}
-
 16%|██████████████████                                                                                               | 400/2499 [47:36<3:39:25,  6.27s/it][2025-12-28 11:53:12,658] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 11:53:14,405] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8942761421203613
-[2025-12-28 11:53:15,270] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8648979663848877
-[2025-12-28 11:53:16,140] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8690180778503418
-[2025-12-28 11:53:16,993] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8520760536193848
-[2025-12-28 11:53:16,993] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
-
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.41it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:55,  1.58it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<01:00,  1.42it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:05,  1.30it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:06,  1.27it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:08,  1.21it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:06<01:07,  1.22it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:08,  1.19it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:06,  1.20it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:07,  1.17it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:05,  1.19it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:06,  1.16it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:04,  1.18it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:04,  1.16it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:12<01:02,  1.18it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:13<01:05,  1.11it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:02,  1.15it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:02,  1.13it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<01:00,  1.16it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<01:00,  1.14it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:58,  1.16it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:19<00:59,  1.13it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:19<00:56,  1.16it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:56,  1.14it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.17it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:54,  1.15it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:52,  1.17it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:53,  1.15it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:25<00:51,  1.17it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:25<00:51,  1.15it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.17it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:49,  1.16it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:47,  1.17it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:47,  1.15it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:46,  1.17it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:46,  1.15it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:31<00:44,  1.17it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:33<00:50,  1.02it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:41,  1.21it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:41,  1.18it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:40,  1.19it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:40,  1.17it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:38,  1.18it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:38<00:38,  1.16it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:38<00:37,  1.18it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.16it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:35,  1.18it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.16it/s]
-
 56%|██████████████████████████████████████████████████████████████��██▌                                                    | 50/90 [00:42<00:33,  1.18it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:33,  1.16it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:43<00:32,  1.18it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:44<00:31,  1.16it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.18it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.15it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:29,  1.17it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.15it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.17it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:26,  1.15it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:50<00:25,  1.17it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:51<00:25,  1.15it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:23,  1.17it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:53<00:23,  1.14it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:22,  1.17it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:21,  1.14it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:20,  1.16it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:57<00:20,  1.14it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:57<00:18,  1.16it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:58<00:18,  1.14it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.17it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.14it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.16it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.14it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.17it/s]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:03<00:13,  1.15it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:04<00:11,  1.17it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:05<00:11,  1.15it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.18it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.15it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.17it/s]
-
 90%|███████████████████████████████████████████��██████████████████████████████████████████████████████████████▏           | 81/90 [01:09<00:07,  1.15it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:09<00:06,  1.18it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:10<00:06,  1.16it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:11<00:05,  1.18it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.16it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.18it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.16it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:15<00:01,  1.18it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89/90 [01:15<00:00,  1.16it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:16<00:00,  1.15it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.5707286596298218, 'eval_runtime': 78.7514, 'eval_samples_per_second': 9.27, 'eval_steps_per_second': 2.324, 'eval_ppl': 1.7696, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.25, 'epoch': 0.48}
-
 16%|██████████████████                                                                                               | 400/2499 [48:59<3:39:25,  6.27s/it]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.15it/s]
-
                                                                                                                                                           [2025-12-28 11:54:39,612] [INFO] [axolotl.core.trainers.base._save:692] [PID:42410] Saving model checkpoint to ./outputs/luau-codellama-h200/checkpoint-400
-
 16%|█████████████████▉                                                                                              | 401/2499 [49:10<19:00:44, 32.62s/it]
                                                                                                                                                           
{'loss': 0.6189, 'grad_norm': 0.16039417684078217, 'learning_rate': 0.0001881269651023858, 'ppl': 1.8569, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.01, 'tokens_per_second_per_gpu': 4650.7, 'total_tokens': 17862360, 'epoch': 0.48}
-
 16%|█████████████████▉                                                                                              | 401/2499 [49:10<19:00:44, 32.62s/it]
 16%|██████████████████                                                                                              | 402/2499 [49:16<14:23:47, 24.72s/it]
                                                                                                                                                           
{'loss': 0.5702, 'grad_norm': 0.15665364265441895, 'learning_rate': 0.0001880672420454887, 'ppl': 1.7686, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.03, 'tokens_per_second_per_gpu': 4577.65, 'total_tokens': 17891015, 'epoch': 0.48}
-
 16%|██████████████████                                                                                              | 402/2499 [49:16<14:23:47, 24.72s/it]
 16%|██████████████████                                                                                              | 403/2499 [49:23<11:10:17, 19.19s/it]
                                                                                                                                                           
{'loss': 0.5665, 'grad_norm': 0.15146903693675995, 'learning_rate': 0.00018800737868645312, 'ppl': 1.7621, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.03, 'tokens_per_second_per_gpu': 4635.76, 'total_tokens': 17920165, 'epoch': 0.48}
-
 16%|██████████████████                                                                                              | 403/2499 [49:23<11:10:17, 19.19s/it]
 16%|██████████████████▎                                                                                              | 404/2499 [49:29<8:54:38, 15.31s/it]
                                                                                                                                                           
{'loss': 0.6125, 'grad_norm': 0.1600656360387802, 'learning_rate': 0.0001879473751206489, 'ppl': 1.845, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4530.87, 'total_tokens': 17948550, 'epoch': 0.48}
-
 16%|██████████████████▎                                                                                              | 404/2499 [49:29<8:54:38, 15.31s/it]
 16%|██████████████████▎                                                                                              | 405/2499 [49:35<7:19:31, 12.59s/it]
                                                                                                                                                           
{'loss': 0.5991, 'grad_norm': 0.17024600505828857, 'learning_rate': 0.00018788723144366927, 'ppl': 1.8205, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4537.43, 'total_tokens': 17976903, 'epoch': 0.49}
-
 16%|██████████████████▎                                                                                              | 405/2499 [49:35<7:19:31, 12.59s/it]
 16%|██████████████████▎                                                                                              | 406/2499 [49:41<6:13:03, 10.69s/it]
                                                                                                                                                           
{'loss': 0.5744, 'grad_norm': 0.17297804355621338, 'learning_rate': 0.00018782694775133058, 'ppl': 1.7761, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4652.34, 'total_tokens': 18006026, 'epoch': 0.49}
-
 16%|██████████████████▎                                                                                              | 406/2499 [49:41<6:13:03, 10.69s/it]
 16%|██████████████████▍                                                                                              | 407/2499 [49:48<5:26:30,  9.36s/it]
                                                                                                                                                           
{'loss': 0.5972, 'grad_norm': 0.17486798763275146, 'learning_rate': 0.00018776652413967236, 'ppl': 1.817, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4501.88, 'total_tokens': 18034203, 'epoch': 0.49}
-
 16%|██████████████████▍                                                                                              | 407/2499 [49:48<5:26:30,  9.36s/it]
 16%|██████████████████▍                                                                                              | 408/2499 [49:54<4:53:56,  8.43s/it]
                                                                                                                                                           
{'loss': 0.4842, 'grad_norm': 0.14348573982715607, 'learning_rate': 0.0001877059607049569, 'ppl': 1.6229, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4636.87, 'total_tokens': 18063240, 'epoch': 0.49}
-
 16%|██████████████████▍                                                                                              | 408/2499 [49:54<4:53:56,  8.43s/it]
 16%|██████████████████▍                                                                                              | 409/2499 [50:00<4:31:29,  7.79s/it]
                                                                                                                                                           
{'loss': 0.5476, 'grad_norm': 0.14626961946487427, 'learning_rate': 0.00018764525754366937, 'ppl': 1.7291, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4564.52, 'total_tokens': 18091984, 'epoch': 0.49}
-
 16%|██████████████████▍                                                                                              | 409/2499 [50:00<4:31:29,  7.79s/it]
 16%|██████████████████▌                                                                                              | 410/2499 [50:07<4:15:24,  7.34s/it]
                                                                                                                                                           
{'loss': 0.5608, 'grad_norm': 0.18620796501636505, 'learning_rate': 0.00018758441475251754, 'ppl': 1.7521, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4481.57, 'total_tokens': 18120048, 'epoch': 0.49}
-
 16%|██████████████████▌                                                                                              | 410/2499 [50:07<4:15:24,  7.34s/it]
 16%|██████████████████▌                                                                                              | 411/2499 [50:13<4:04:07,  7.02s/it]
                                                                                                                                                           
{'loss': 0.5314, 'grad_norm': 0.15043221414089203, 'learning_rate': 0.00018752343242843154, 'ppl': 1.7013, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4554.39, 'total_tokens': 18148581, 'epoch': 0.49}
-
 16%|██████████████████▌                                                                                              | 411/2499 [50:13<4:04:07,  7.02s/it]
 16%|██████████████████▋                                                                                              | 412/2499 [50:19<3:56:08,  6.79s/it]
                                                                                                                                                           
{'loss': 0.6299, 'grad_norm': 0.15692859888076782, 'learning_rate': 0.00018746231066856387, 'ppl': 1.8774, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4626.86, 'total_tokens': 18177538, 'epoch': 0.49}
-
 16%|██████████████████▋                                                                                              | 412/2499 [50:19<3:56:08,  6.79s/it]
 17%|██████████████████▋                                                                                              | 413/2499 [50:25<3:50:31,  6.63s/it]
                                                                                                                                                           
{'loss': 0.6237, 'grad_norm': 0.15896819531917572, 'learning_rate': 0.00018740104957028913, 'ppl': 1.8658, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4577.2, 'total_tokens': 18206187, 'epoch': 0.5}
-
 17%|██████████████████▋                                                                                              | 413/2499 [50:25<3:50:31,  6.63s/it]
 17%|██████████████████▋                                                                                              | 414/2499 [50:32<3:46:30,  6.52s/it]
                                                                                                                                                           
{'loss': 0.65, 'grad_norm': 0.18454909324645996, 'learning_rate': 0.00018733964923120392, 'ppl': 1.9155, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4508.93, 'total_tokens': 18234384, 'epoch': 0.5}
-
 17%|██████████████████▋                                                                                              | 414/2499 [50:32<3:46:30,  6.52s/it]
 17%|██████████████████▊                                                                                              | 415/2499 [50:38<3:44:02,  6.45s/it]
                                                                                                                                                           
{'loss': 0.5645, 'grad_norm': 0.15952667593955994, 'learning_rate': 0.0001872781097491267, 'ppl': 1.7586, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4557.17, 'total_tokens': 18263045, 'epoch': 0.5}
-
 17%|██████████████████▊                                                                                              | 415/2499 [50:38<3:44:02,  6.45s/it]
 17%|██████████████████▊                                                                                              | 416/2499 [50:44<3:41:57,  6.39s/it]
                                                                                                                                                           
{'loss': 0.567, 'grad_norm': 0.15676908195018768, 'learning_rate': 0.0001872164312220975, 'ppl': 1.763, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4339.73, 'total_tokens': 18290199, 'epoch': 0.5}
-
 17%|██████████████████▊                                                                                              | 416/2499 [50:44<3:41:57,  6.39s/it]
 17%|██████████████████▊                                                                                              | 417/2499 [50:50<3:40:27,  6.35s/it]
                                                                                                                                                           
{'loss': 0.5373, 'grad_norm': 0.16627971827983856, 'learning_rate': 0.000187154613748378, 'ppl': 1.7114, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4461.35, 'total_tokens': 18318111, 'epoch': 0.5}
-
 17%|██████████████████▊                                                                                              | 417/2499 [50:50<3:40:27,  6.35s/it]
 17%|██████████████████▉                                                                                              | 418/2499 [50:57<3:39:31,  6.33s/it]
                                                                                                                                                           
{'loss': 0.5771, 'grad_norm': 0.15211078524589539, 'learning_rate': 0.0001870926574264511, 'ppl': 1.7809, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4570.21, 'total_tokens': 18346773, 'epoch': 0.5}
-
 17%|██████████████████▉                                                                                              | 418/2499 [50:57<3:39:31,  6.33s/it]
 17%|██████████████████▉                                                                                              | 419/2499 [51:03<3:38:46,  6.31s/it]
                                                                                                                                                           
{'loss': 0.5499, 'grad_norm': 0.15301309525966644, 'learning_rate': 0.00018703056235502103, 'ppl': 1.7331, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4536.51, 'total_tokens': 18375194, 'epoch': 0.5}
-
 17%|██████████████████▉                                                                                              | 419/2499 [51:03<3:38:46,  6.31s/it]
 17%|██████████████████▉                                                                                              | 420/2499 [51:09<3:38:10,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5583, 'grad_norm': 0.14875005185604095, 'learning_rate': 0.000186968328633013, 'ppl': 1.7477, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4548.16, 'total_tokens': 18403674, 'epoch': 0.5}
-
 17%|██████████████████▉                                                                                              | 420/2499 [51:09<3:38:10,  6.30s/it]
 17%|███████████████████                                                                                              | 421/2499 [51:16<3:37:58,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5516, 'grad_norm': 0.16316725313663483, 'learning_rate': 0.00018690595635957312, 'ppl': 1.736, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4436.18, 'total_tokens': 18431553, 'epoch': 0.51}
-
 17%|███████████████████                                                                                              | 421/2499 [51:16<3:37:58,  6.29s/it]
 17%|███████████████████                                                                                              | 422/2499 [51:22<3:38:26,  6.31s/it]
                                                                                                                                                           
{'loss': 0.5639, 'grad_norm': 0.15915672481060028, 'learning_rate': 0.0001868434456340682, 'ppl': 1.7575, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4741.62, 'total_tokens': 18461634, 'epoch': 0.51}
-
 17%|███████████████████                                                                                              | 422/2499 [51:22<3:38:26,  6.31s/it]
 17%|███████████████████▏                                                                                             | 423/2499 [51:28<3:37:56,  6.30s/it]
                                                                                                                                                           
{'loss': 0.623, 'grad_norm': 0.15488934516906738, 'learning_rate': 0.00018678079655608568, 'ppl': 1.8645, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4564.65, 'total_tokens': 18490250, 'epoch': 0.51}
-
 17%|███████████████████▏                                                                                             | 423/2499 [51:28<3:37:56,  6.30s/it]
 17%|███████████████████▏                                                                                             | 424/2499 [51:34<3:37:33,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5389, 'grad_norm': 0.16349388659000397, 'learning_rate': 0.00018671800922543338, 'ppl': 1.7141, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4610.8, 'total_tokens': 18519157, 'epoch': 0.51}
-
 17%|███████████████████▏                                                                                             | 424/2499 [51:34<3:37:33,  6.29s/it]
 17%|███████████████████▏                                                                                             | 425/2499 [51:41<3:37:23,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5917, 'grad_norm': 0.17400610446929932, 'learning_rate': 0.00018665508374213937, 'ppl': 1.8071, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4723.73, 'total_tokens': 18548836, 'epoch': 0.51}
-
 17%|███████████████████▏                                                                                             | 425/2499 [51:41<3:37:23,  6.29s/it]
 17%|███████████████████▎                                                                                             | 426/2499 [51:47<3:37:05,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5805, 'grad_norm': 0.15097637474536896, 'learning_rate': 0.00018659202020645182, 'ppl': 1.7869, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4556.88, 'total_tokens': 18577399, 'epoch': 0.51}
-
 17%|███████████████████▎                                                                                             | 426/2499 [51:47<3:37:05,  6.28s/it]
 17%|███████████████████▎                                                                                             | 427/2499 [51:53<3:36:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.476, 'grad_norm': 0.13820724189281464, 'learning_rate': 0.0001865288187188388, 'ppl': 1.6096, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4660.43, 'total_tokens': 18606658, 'epoch': 0.51}
-
 17%|███████████████████▎                                                                                             | 427/2499 [51:53<3:36:57,  6.28s/it]
 17%|███████████████████▎                                                                                             | 428/2499 [52:00<3:36:59,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5665, 'grad_norm': 0.1509668529033661, 'learning_rate': 0.00018646547937998826, 'ppl': 1.7621, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4493.09, 'total_tokens': 18634930, 'epoch': 0.51}
-
 17%|███████████████████▎                                                                                             | 428/2499 [52:00<3:36:59,  6.29s/it]
 17%|███████████████████▍                                                                                             | 429/2499 [52:06<3:37:16,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6098, 'grad_norm': 0.16413377225399017, 'learning_rate': 0.00018640200229080763, 'ppl': 1.8401, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4687.87, 'total_tokens': 18664562, 'epoch': 0.52}
-
 17%|███████████████████▍                                                                                             | 429/2499 [52:06<3:37:16,  6.30s/it]
 17%|███████████████████▍                                                                                             | 430/2499 [52:12<3:37:06,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5941, 'grad_norm': 0.15067212283611298, 'learning_rate': 0.00018633838755242389, 'ppl': 1.8114, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4641.44, 'total_tokens': 18693744, 'epoch': 0.52}
-
 17%|███████████████████▍                                                                                             | 430/2499 [52:12<3:37:06,  6.30s/it]
 17%|███████████████████▍                                                                                             | 431/2499 [52:18<3:36:44,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6031, 'grad_norm': 0.16875723004341125, 'learning_rate': 0.00018627463526618327, 'ppl': 1.8278, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4484.05, 'total_tokens': 18721853, 'epoch': 0.52}
-
 17%|███████████████████▍                                                                                             | 431/2499 [52:18<3:36:44,  6.29s/it]
 17%|███████████████████▌                                                                                             | 432/2499 [52:25<3:36:23,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5569, 'grad_norm': 0.16489025950431824, 'learning_rate': 0.00018621074553365117, 'ppl': 1.7453, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4513.86, 'total_tokens': 18750119, 'epoch': 0.52}
-
 17%|███████████████████▌                                                                                             | 432/2499 [52:25<3:36:23,  6.28s/it]
 17%|███████████████████▌                                                                                             | 433/2499 [52:31<3:36:14,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6399, 'grad_norm': 0.16152667999267578, 'learning_rate': 0.0001861467184566119, 'ppl': 1.8963, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4645.4, 'total_tokens': 18779267, 'epoch': 0.52}
-
 17%|███████████████████▌                                                                                             | 433/2499 [52:31<3:36:14,  6.28s/it]
 17%|███████████████████▌                                                                                             | 434/2499 [52:37<3:36:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5099, 'grad_norm': 0.15479077398777008, 'learning_rate': 0.0001860825541370686, 'ppl': 1.6651, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4642.91, 'total_tokens': 18808360, 'epoch': 0.52}
-
 17%|███████████████████▌                                                                                             | 434/2499 [52:37<3:36:01,  6.28s/it]
 17%|███████████████████▋                                                                                             | 435/2499 [52:44<3:36:14,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5588, 'grad_norm': 0.14588800072669983, 'learning_rate': 0.00018601825267724307, 'ppl': 1.7486, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4598.14, 'total_tokens': 18837352, 'epoch': 0.52}
-
 17%|███████████████████▋                                                                                             | 435/2499 [52:44<3:36:14,  6.29s/it]
 17%|███████████████████▋                                                                                             | 436/2499 [52:50<3:36:24,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5802, 'grad_norm': 0.14621266722679138, 'learning_rate': 0.00018595381417957558, 'ppl': 1.7864, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4486.83, 'total_tokens': 18865660, 'epoch': 0.52}
-
 17%|███████████████████▋                                                                                             | 436/2499 [52:50<3:36:24,  6.29s/it]
 17%|███████████████████▊                                                                                             | 437/2499 [52:56<3:36:07,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5689, 'grad_norm': 0.1723642498254776, 'learning_rate': 0.00018588923874672474, 'ppl': 1.7663, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4501.25, 'total_tokens': 18893893, 'epoch': 0.52}
-
 17%|███████████████████▊                                                                                             | 437/2499 [52:56<3:36:07,  6.29s/it]
 18%|███████████████████▊                                                                                             | 438/2499 [53:02<3:35:50,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5598, 'grad_norm': 0.14782671630382538, 'learning_rate': 0.00018582452648156726, 'ppl': 1.7503, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4625.08, 'total_tokens': 18922889, 'epoch': 0.53}
-
 18%|███████████████████▊                                                                                             | 438/2499 [53:02<3:35:50,  6.28s/it]
 18%|███████████████████▊                                                                                             | 439/2499 [53:09<3:35:38,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6162, 'grad_norm': 0.16162589192390442, 'learning_rate': 0.0001857596774871979, 'ppl': 1.8519, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4605.11, 'total_tokens': 18951769, 'epoch': 0.53}
-
 18%|███████████████████▊                                                                                             | 439/2499 [53:09<3:35:38,  6.28s/it]
 18%|███████████████████▉                                                                                             | 440/2499 [53:15<3:35:36,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5212, 'grad_norm': 0.15044647455215454, 'learning_rate': 0.00018569469186692925, 'ppl': 1.684, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4708.11, 'total_tokens': 18981362, 'epoch': 0.53}
-
 18%|███████████████████▉                                                                                             | 440/2499 [53:15<3:35:36,  6.28s/it]
 18%|███████████████████▉                                                                                             | 441/2499 [53:21<3:35:23,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5599, 'grad_norm': 0.1452936828136444, 'learning_rate': 0.0001856295697242915, 'ppl': 1.7505, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4647.74, 'total_tokens': 19010504, 'epoch': 0.53}
-
 18%|███████████████████▉                                                                                             | 441/2499 [53:21<3:35:23,  6.28s/it]
 18%|███████████████████▉                                                                                             | 442/2499 [53:28<3:35:26,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5611, 'grad_norm': 0.1456039994955063, 'learning_rate': 0.0001855643111630324, 'ppl': 1.7526, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4472.27, 'total_tokens': 19038640, 'epoch': 0.53}
-
 18%|███████████████████▉                                                                                             | 442/2499 [53:28<3:35:26,  6.28s/it]
 18%|████████████████████                                                                                             | 443/2499 [53:34<3:35:20,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5572, 'grad_norm': 0.15803247690200806, 'learning_rate': 0.00018549891628711696, 'ppl': 1.7458, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4579.12, 'total_tokens': 19067400, 'epoch': 0.53}
-
 18%|████████████████████                                                                                             | 443/2499 [53:34<3:35:20,  6.28s/it]
 18%|████████████████████                                                                                             | 444/2499 [53:40<3:35:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5711, 'grad_norm': 0.15013474225997925, 'learning_rate': 0.00018543338520072745, 'ppl': 1.7702, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4475.98, 'total_tokens': 19095424, 'epoch': 0.53}
-
 18%|████████████████████                                                                                             | 444/2499 [53:40<3:35:01,  6.28s/it]
 18%|████████████████████                                                                                             | 445/2499 [53:46<3:34:47,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5748, 'grad_norm': 0.16678114235401154, 'learning_rate': 0.00018536771800826304, 'ppl': 1.7768, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4595.63, 'total_tokens': 19124207, 'epoch': 0.53}
-
 18%|████████████████████                                                                                             | 445/2499 [53:46<3:34:47,  6.27s/it]
 18%|████████████████████▏                                                                                            | 446/2499 [53:53<3:34:34,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5822, 'grad_norm': 0.15794029831886292, 'learning_rate': 0.00018530191481433986, 'ppl': 1.79, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4661.59, 'total_tokens': 19153399, 'epoch': 0.54}
-
 18%|████████████████████▏                                                                                            | 446/2499 [53:53<3:34:34,  6.27s/it]
 18%|████████████████████▏                                                                                            | 447/2499 [53:59<3:34:22,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5633, 'grad_norm': 0.15021128952503204, 'learning_rate': 0.0001852359757237906, 'ppl': 1.7565, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4615.87, 'total_tokens': 19182290, 'epoch': 0.54}
-
 18%|████████████████████▏                                                                                            | 447/2499 [53:59<3:34:22,  6.27s/it]
 18%|████████████████████▎                                                                                            | 448/2499 [54:05<3:34:36,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5825, 'grad_norm': 0.16197733581066132, 'learning_rate': 0.00018516990084166442, 'ppl': 1.7905, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4640.39, 'total_tokens': 19211516, 'epoch': 0.54}
-
 18%|████████████████████▎                                                                                            | 448/2499 [54:05<3:34:36,  6.28s/it]
 18%|████████████████████▎                                                                                            | 449/2499 [54:11<3:34:56,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6094, 'grad_norm': 0.1648341864347458, 'learning_rate': 0.000185103690273227, 'ppl': 1.8393, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4526.22, 'total_tokens': 19240112, 'epoch': 0.54}
-
 18%|████████████████████▎                                                                                            | 449/2499 [54:11<3:34:56,  6.29s/it]
 18%|████████████████████▎                                                                                            | 450/2499 [54:18<3:34:49,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5692, 'grad_norm': 0.15157613158226013, 'learning_rate': 0.00018503734412395994, 'ppl': 1.7669, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4723.23, 'total_tokens': 19269798, 'epoch': 0.54}
-
 18%|████████████████████▎                                                                                            | 450/2499 [54:18<3:34:49,  6.29s/it]
 18%|████████████████████▍                                                                                            | 451/2499 [54:24<3:34:38,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5899, 'grad_norm': 0.15227428078651428, 'learning_rate': 0.00018497086249956107, 'ppl': 1.8038, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4752.62, 'total_tokens': 19299647, 'epoch': 0.54}
-
 18%|████████████████████▍                                                                                            | 451/2499 [54:24<3:34:38,  6.29s/it]
 18%|████████████████████▍                                                                                            | 452/2499 [54:30<3:34:18,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5725, 'grad_norm': 0.18285728991031647, 'learning_rate': 0.00018490424550594384, 'ppl': 1.7727, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4465.02, 'total_tokens': 19327610, 'epoch': 0.54}
-
 18%|████████████████████▍                                                                                            | 452/2499 [54:30<3:34:18,  6.28s/it]
 18%|████████████████████▍                                                                                            | 453/2499 [54:37<3:34:07,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5944, 'grad_norm': 0.1537967026233673, 'learning_rate': 0.00018483749324923752, 'ppl': 1.8119, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4686.13, 'total_tokens': 19357004, 'epoch': 0.54}
-
 18%|████████████████████▍                                                                                            | 453/2499 [54:37<3:34:07,  6.28s/it]
 18%|████████████████████▌                                                                                            | 454/2499 [54:43<3:33:52,  6.28s/it]
                                                                                                                                                           
{'loss': 0.605, 'grad_norm': 0.16797775030136108, 'learning_rate': 0.00018477060583578676, 'ppl': 1.8313, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4550.85, 'total_tokens': 19385505, 'epoch': 0.55}
-
 18%|████████████████████▌                                                                                            | 454/2499 [54:43<3:33:52,  6.28s/it]
 18%|████████████████████▌                                                                                            | 455/2499 [54:49<3:33:56,  6.28s/it]
                                                                                                                                                           
{'loss': 0.621, 'grad_norm': 0.1565423309803009, 'learning_rate': 0.00018470358337215162, 'ppl': 1.8608, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4593.26, 'total_tokens': 19414395, 'epoch': 0.55}
-
 18%|████████████████████▌                                                                                            | 455/2499 [54:49<3:33:56,  6.28s/it]
 18%|████████████████████▌                                                                                            | 456/2499 [54:55<3:34:10,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5541, 'grad_norm': 0.15655626356601715, 'learning_rate': 0.0001846364259651073, 'ppl': 1.7404, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4694.8, 'total_tokens': 19444018, 'epoch': 0.55}
-
 18%|████████████████████▌                                                                                            | 456/2499 [54:55<3:34:10,  6.29s/it]
 18%|████████████████████▋                                                                                            | 457/2499 [55:02<3:33:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5553, 'grad_norm': 0.15646992623806, 'learning_rate': 0.00018456913372164388, 'ppl': 1.7425, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4453.86, 'total_tokens': 19471900, 'epoch': 0.55}
-
 18%|████████████████████▋                                                                                            | 457/2499 [55:02<3:33:47,  6.28s/it]
 18%|████████████████████▋                                                                                            | 458/2499 [55:08<3:33:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5948, 'grad_norm': 0.1484660506248474, 'learning_rate': 0.0001845017067489664, 'ppl': 1.8127, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4470.62, 'total_tokens': 19499866, 'epoch': 0.55}
-
 18%|████████████████████▋                                                                                            | 458/2499 [55:08<3:33:26,  6.27s/it]
 18%|████████████████████▊                                                                                            | 459/2499 [55:14<3:33:14,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5839, 'grad_norm': 0.1672670543193817, 'learning_rate': 0.00018443414515449438, 'ppl': 1.793, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4569.68, 'total_tokens': 19528485, 'epoch': 0.55}
-
 18%|████████████████████▊                                                                                            | 459/2499 [55:14<3:33:14,  6.27s/it]
 18%|████████████████████▊                                                                                            | 460/2499 [55:21<3:33:01,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6341, 'grad_norm': 0.1631045639514923, 'learning_rate': 0.00018436644904586198, 'ppl': 1.8853, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4502.65, 'total_tokens': 19556666, 'epoch': 0.55}
-
 18%|████████████████████▊                                                                                            | 460/2499 [55:21<3:33:01,  6.27s/it]
 18%|████████████████████▊                                                                                            | 461/2499 [55:27<3:32:52,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5554, 'grad_norm': 0.15775103867053986, 'learning_rate': 0.00018429861853091754, 'ppl': 1.7426, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4483.55, 'total_tokens': 19584739, 'epoch': 0.55}
-
 18%|████████████████████▊                                                                                            | 461/2499 [55:27<3:32:52,  6.27s/it]
 18%|████████████████████▉                                                                                            | 462/2499 [55:33<3:33:02,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5995, 'grad_norm': 0.16724328696727753, 'learning_rate': 0.00018423065371772355, 'ppl': 1.8212, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4399.67, 'total_tokens': 19612415, 'epoch': 0.55}
-
 18%|████████████████████▉                                                                                            | 462/2499 [55:33<3:33:02,  6.27s/it]
 19%|████████████████████▉                                                                                            | 463/2499 [55:39<3:33:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5533, 'grad_norm': 0.14767299592494965, 'learning_rate': 0.00018416255471455646, 'ppl': 1.739, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4603.33, 'total_tokens': 19641326, 'epoch': 0.56}
-
 19%|████████████████████▉                                                                                            | 463/2499 [55:39<3:33:01,  6.28s/it]
 19%|████████████████████▉                                                                                            | 464/2499 [55:46<3:32:50,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5469, 'grad_norm': 0.16348305344581604, 'learning_rate': 0.0001840943216299065, 'ppl': 1.7279, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4562.37, 'total_tokens': 19669924, 'epoch': 0.56}
-
 19%|████████████████████▉                                                                                            | 464/2499 [55:46<3:32:50,  6.28s/it]
 19%|█████████████████████                                                                                            | 465/2499 [55:52<3:32:48,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5275, 'grad_norm': 0.15997561812400818, 'learning_rate': 0.00018402595457247758, 'ppl': 1.6947, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4670.61, 'total_tokens': 19699254, 'epoch': 0.56}
-
 19%|█████████████████████                                                                                            | 465/2499 [55:52<3:32:48,  6.28s/it]
 19%|█████████████████████                                                                                            | 466/2499 [55:58<3:32:25,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5625, 'grad_norm': 0.16886287927627563, 'learning_rate': 0.00018395745365118687, 'ppl': 1.7551, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4419.59, 'total_tokens': 19726868, 'epoch': 0.56}
-
 19%|█████████████████████                                                                                            | 466/2499 [55:58<3:32:25,  6.27s/it]
 19%|█████████████████████                                                                                            | 467/2499 [56:04<3:32:19,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5631, 'grad_norm': 0.14943641424179077, 'learning_rate': 0.000183888818975165, 'ppl': 1.7561, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4526.6, 'total_tokens': 19755240, 'epoch': 0.56}
-
 19%|█████████████████████                                                                                            | 467/2499 [56:04<3:32:19,  6.27s/it]
 19%|█████████████████████▏                                                                                           | 468/2499 [56:11<3:32:39,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5527, 'grad_norm': 0.1644650548696518, 'learning_rate': 0.0001838200506537556, 'ppl': 1.7379, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4594.43, 'total_tokens': 19784231, 'epoch': 0.56}
-
 19%|█████████████████████▏                                                                                           | 468/2499 [56:11<3:32:39,  6.28s/it]
 19%|█████████████████████▏                                                                                           | 469/2499 [56:17<3:32:52,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5602, 'grad_norm': 0.21660394966602325, 'learning_rate': 0.0001837511487965151, 'ppl': 1.751, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4405.97, 'total_tokens': 19812033, 'epoch': 0.56}
-
 19%|█████████████████████▏                                                                                           | 469/2499 [56:17<3:32:52,  6.29s/it]
 19%|█████████████████████▎                                                                                           | 470/2499 [56:23<3:32:19,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5774, 'grad_norm': 0.16006816923618317, 'learning_rate': 0.00018368211351321294, 'ppl': 1.7814, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4336.95, 'total_tokens': 19839111, 'epoch': 0.56}
-
 19%|█████████████████████▎                                                                                           | 470/2499 [56:23<3:32:19,  6.28s/it]
 19%|█████████████████████▎                                                                                           | 471/2499 [56:30<3:32:05,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6026, 'grad_norm': 0.15922212600708008, 'learning_rate': 0.00018361294491383085, 'ppl': 1.8269, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4576.39, 'total_tokens': 19867776, 'epoch': 0.57}
-
 19%|█████████████████████▎                                                                                           | 471/2499 [56:30<3:32:05,  6.27s/it]
 19%|█████████████████████▎                                                                                           | 472/2499 [56:36<3:32:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5705, 'grad_norm': 0.15619011223316193, 'learning_rate': 0.0001835436431085631, 'ppl': 1.7692, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4555.67, 'total_tokens': 19896367, 'epoch': 0.57}
-
 19%|█████████████████████▎                                                                                           | 472/2499 [56:36<3:32:01,  6.28s/it]
 19%|█████████████████████▍                                                                                           | 473/2499 [56:42<3:31:52,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5242, 'grad_norm': 0.16088345646858215, 'learning_rate': 0.00018347420820781618, 'ppl': 1.6891, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4618.23, 'total_tokens': 19925320, 'epoch': 0.57}
-
 19%|█████████████████████▍                                                                                           | 473/2499 [56:42<3:31:52,  6.27s/it]
 19%|█████████████████████▍                                                                                           | 474/2499 [56:48<3:31:43,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5705, 'grad_norm': 0.14945322275161743, 'learning_rate': 0.0001834046403222085, 'ppl': 1.7692, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4559.94, 'total_tokens': 19953900, 'epoch': 0.57}
-
 19%|█████████████████████▍                                                                                           | 474/2499 [56:48<3:31:43,  6.27s/it]
 19%|█████████████████████▍                                                                                           | 475/2499 [56:55<3:31:55,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5825, 'grad_norm': 0.16998130083084106, 'learning_rate': 0.0001833349395625705, 'ppl': 1.7905, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4578.29, 'total_tokens': 19982746, 'epoch': 0.57}
-
 19%|█████████████████████▍                                                                                           | 475/2499 [56:55<3:31:55,  6.28s/it]
 19%|█████████████████████▌                                                                                           | 476/2499 [57:01<3:32:08,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5645, 'grad_norm': 0.16215763986110687, 'learning_rate': 0.00018326510603994408, 'ppl': 1.7586, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4533.6, 'total_tokens': 20011356, 'epoch': 0.57}
-
 19%|█████████████████████▌                                                                                           | 476/2499 [57:01<3:32:08,  6.29s/it]
 19%|█████████████████████▌                                                                                           | 477/2499 [57:07<3:31:57,  6.29s/it]
                                                                                                                                                           
{'loss': 0.517, 'grad_norm': 0.1561872959136963, 'learning_rate': 0.0001831951398655829, 'ppl': 1.677, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4588.7, 'total_tokens': 20040170, 'epoch': 0.57}
-
 19%|█████████████████████▌                                                                                           | 477/2499 [57:07<3:31:57,  6.29s/it]
 19%|█████████████████████▌                                                                                           | 478/2499 [57:14<3:31:45,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5762, 'grad_norm': 0.14602787792682648, 'learning_rate': 0.00018312504115095183, 'ppl': 1.7793, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4743.0, 'total_tokens': 20069945, 'epoch': 0.57}
-
 19%|█████████████████████▌                                                                                           | 478/2499 [57:14<3:31:45,  6.29s/it]
 19%|█████████████████████▋                                                                                           | 479/2499 [57:20<3:31:22,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5952, 'grad_norm': 0.15776875615119934, 'learning_rate': 0.0001830548100077268, 'ppl': 1.8134, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4432.95, 'total_tokens': 20097685, 'epoch': 0.58}
-
 19%|█████████████████████▋                                                                                           | 479/2499 [57:20<3:31:22,  6.28s/it]
 19%|█████████████████████▋                                                                                           | 480/2499 [57:26<3:31:12,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5511, 'grad_norm': 0.17158068716526031, 'learning_rate': 0.00018298444654779494, 'ppl': 1.7352, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4570.76, 'total_tokens': 20126338, 'epoch': 0.58}
-
 19%|█████████████████████▋                                                                                           | 480/2499 [57:26<3:31:12,  6.28s/it]
 19%|█████████████████████▋                                                                                           | 481/2499 [57:32<3:30:57,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5418, 'grad_norm': 0.16042540967464447, 'learning_rate': 0.00018291395088325393, 'ppl': 1.7191, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4549.64, 'total_tokens': 20154820, 'epoch': 0.58}
-
 19%|█████████████████████▋                                                                                           | 481/2499 [57:32<3:30:57,  6.27s/it]
 19%|█████████████████████▊                                                                                           | 482/2499 [57:39<3:31:05,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5878, 'grad_norm': 0.15308934450149536, 'learning_rate': 0.00018284332312641226, 'ppl': 1.8, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4432.0, 'total_tokens': 20182711, 'epoch': 0.58}
-
 19%|█████████████████████▊                                                                                           | 482/2499 [57:39<3:31:05,  6.28s/it]
 19%|█████████████████████▊                                                                                           | 483/2499 [57:45<3:31:21,  6.29s/it]
                                                                                                                                                           
{'loss': 0.571, 'grad_norm': 0.1654343158006668, 'learning_rate': 0.00018277256338978875, 'ppl': 1.77, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4780.75, 'total_tokens': 20212891, 'epoch': 0.58}
-
 19%|█████████████████████▊                                                                                           | 483/2499 [57:45<3:31:21,  6.29s/it]
 19%|█████████████████████▉                                                                                           | 484/2499 [57:51<3:31:13,  6.29s/it]
                                                                                                                                                           
{'loss': 0.62, 'grad_norm': 0.15104906260967255, 'learning_rate': 0.00018270167178611254, 'ppl': 1.8589, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4787.43, 'total_tokens': 20242973, 'epoch': 0.58}
-
 19%|█████████████████████▉                                                                                           | 484/2499 [57:51<3:31:13,  6.29s/it]
 19%|█████████████████████▉                                                                                           | 485/2499 [57:58<3:30:54,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5792, 'grad_norm': 0.15305499732494354, 'learning_rate': 0.00018263064842832281, 'ppl': 1.7846, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4599.65, 'total_tokens': 20271797, 'epoch': 0.58}
-
 19%|█████████████████████▉                                                                                           | 485/2499 [57:58<3:30:54,  6.28s/it]
 19%|█████████████████████▉                                                                                           | 486/2499 [58:04<3:30:32,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6147, 'grad_norm': 0.16085773706436157, 'learning_rate': 0.00018255949342956863, 'ppl': 1.8491, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4499.19, 'total_tokens': 20299936, 'epoch': 0.58}
-
 19%|█████████████████████▉                                                                                           | 486/2499 [58:04<3:30:32,  6.28s/it]
 19%|██████████████████████                                                                                           | 487/2499 [58:10<3:30:11,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6101, 'grad_norm': 0.18756824731826782, 'learning_rate': 0.00018248820690320889, 'ppl': 1.8406, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4454.08, 'total_tokens': 20327766, 'epoch': 0.58}
-
 19%|██████████████████████                                                                                           | 487/2499 [58:10<3:30:11,  6.27s/it]
 20%|██████████████████████                                                                                           | 488/2499 [58:16<3:30:03,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6178, 'grad_norm': 0.17079570889472961, 'learning_rate': 0.00018241678896281188, 'ppl': 1.8548, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4494.52, 'total_tokens': 20355914, 'epoch': 0.59}
-
 20%|██████████████████████                                                                                           | 488/2499 [58:16<3:30:03,  6.27s/it]
 20%|██████████████████████                                                                                           | 489/2499 [58:23<3:30:06,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6194, 'grad_norm': 0.16470293700695038, 'learning_rate': 0.00018234523972215536, 'ppl': 1.8578, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4335.83, 'total_tokens': 20383145, 'epoch': 0.59}
-
 20%|██████████████████████                                                                                           | 489/2499 [58:23<3:30:06,  6.27s/it]
 20%|██████████████████████▏                                                                                          | 490/2499 [58:29<3:30:02,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5485, 'grad_norm': 0.16623155772686005, 'learning_rate': 0.00018227355929522623, 'ppl': 1.7307, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4540.41, 'total_tokens': 20411623, 'epoch': 0.59}
-
 20%|██████████████████████▏                                                                                          | 490/2499 [58:29<3:30:02,  6.27s/it]
 20%|██████████████████████▏                                                                                          | 491/2499 [58:35<3:30:05,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5823, 'grad_norm': 0.18991202116012573, 'learning_rate': 0.00018220174779622034, 'ppl': 1.7902, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4712.78, 'total_tokens': 20441243, 'epoch': 0.59}
-
 20%|██████████████████████▏                                                                                          | 491/2499 [58:35<3:30:05,  6.28s/it]
 20%|██████████████████████▏                                                                                          | 492/2499 [58:41<3:29:51,  6.27s/it]
                                                                                                                                                           
{'loss': 0.567, 'grad_norm': 0.16380782425403595, 'learning_rate': 0.00018212980533954243, 'ppl': 1.763, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4455.55, 'total_tokens': 20469147, 'epoch': 0.59}
-
 20%|██████████████████████▏                                                                                          | 492/2499 [58:41<3:29:51,  6.27s/it]
 20%|██████████████████████▎                                                                                          | 493/2499 [58:48<3:29:39,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5529, 'grad_norm': 0.15777407586574554, 'learning_rate': 0.00018205773203980582, 'ppl': 1.7383, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4500.07, 'total_tokens': 20497328, 'epoch': 0.59}
-
 20%|██████████████████████▎                                                                                          | 493/2499 [58:48<3:29:39,  6.27s/it]
 20%|██████████████████████▎                                                                                          | 494/2499 [58:54<3:29:43,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5155, 'grad_norm': 0.15534964203834534, 'learning_rate': 0.0001819855280118323, 'ppl': 1.6745, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4716.79, 'total_tokens': 20526972, 'epoch': 0.59}
-
 20%|██████████████████████▎                                                                                          | 494/2499 [58:54<3:29:43,  6.28s/it]
 20%|██████████████████████▍                                                                                          | 495/2499 [59:00<3:29:32,  6.27s/it]
                                                                                                                                                           
{'loss': 0.575, 'grad_norm': 0.1571529060602188, 'learning_rate': 0.00018191319337065195, 'ppl': 1.7771, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4597.26, 'total_tokens': 20555776, 'epoch': 0.59}
-
 20%|██████████████████████▍                                                                                          | 495/2499 [59:00<3:29:32,  6.27s/it]
 20%|██████████████████████▍                                                                                          | 496/2499 [59:07<3:29:34,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5403, 'grad_norm': 0.15741947293281555, 'learning_rate': 0.00018184072823150283, 'ppl': 1.7165, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4368.14, 'total_tokens': 20583232, 'epoch': 0.6}
-
 20%|██████████████████████▍                                                                                          | 496/2499 [59:07<3:29:34,  6.28s/it]
 20%|██████████████████████▍                                                                                          | 497/2499 [59:13<3:29:42,  6.28s/it]
                                                                                                                                                           
{'loss': 0.598, 'grad_norm': 0.15272513031959534, 'learning_rate': 0.00018176813270983107, 'ppl': 1.8185, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4795.9, 'total_tokens': 20613435, 'epoch': 0.6}
-
 20%|██████████████████████▍                                                                                          | 497/2499 [59:13<3:29:42,  6.28s/it]
 20%|██████████████████████▌                                                                                          | 498/2499 [59:19<3:29:15,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5788, 'grad_norm': 0.17274294793605804, 'learning_rate': 0.00018169540692129034, 'ppl': 1.7839, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4501.35, 'total_tokens': 20641562, 'epoch': 0.6}
-
 20%|██████████████████████▌                                                                                          | 498/2499 [59:19<3:29:15,  6.27s/it]
 20%|██████████████████████▌                                                                                          | 499/2499 [59:25<3:29:06,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5548, 'grad_norm': 0.1560908406972885, 'learning_rate': 0.000181622550981742, 'ppl': 1.7416, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4626.44, 'total_tokens': 20670558, 'epoch': 0.6}
-
 20%|██████████████████████▌                                                                                          | 499/2499 [59:25<3:29:06,  6.27s/it]
 20%|██████████████████████▌                                                                                          | 500/2499 [59:32<3:28:52,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5454, 'grad_norm': 0.15503084659576416, 'learning_rate': 0.0001815495650072546, 'ppl': 1.7253, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.15, 'tokens_per_second_per_gpu': 4517.68, 'total_tokens': 20698829, 'epoch': 0.6}
-
 20%|██████████████████████▌                                                                                          | 500/2499 [59:32<3:28:52,  6.27s/it][2025-12-28 12:05:08,249] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 12:05:09,958] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8499741554260254
-[2025-12-28 12:05:10,798] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8398079872131348
-[2025-12-28 12:05:11,648] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.847926139831543
-[2025-12-28 12:05:12,486] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8376157283782959
-[2025-12-28 12:05:12,486] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
+[2025-12-29 02:49:33,870] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:3751] after adapters 0.000GB ()
+[2025-12-29 02:49:42,405] [INFO] [axolotl.train.save_initial_configs:413] [PID:3751] Pre-saving adapter config to ./outputs/luau-codellama-h200-fast...
+[2025-12-29 02:49:42,405] [INFO] [axolotl.train.save_initial_configs:417] [PID:3751] Pre-saving tokenizer to ./outputs/luau-codellama-h200-fast...
+[2025-12-29 02:49:42,407] [INFO] [axolotl.train.save_initial_configs:422] [PID:3751] Pre-saving model config to ./outputs/luau-codellama-h200-fast...
+[2025-12-29 02:49:42,409] [INFO] [axolotl.train.execute_training:212] [PID:3751] Starting trainer...
+[2025-12-29 02:49:43,888] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.4409146308898926
+[2025-12-29 02:49:44,338] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.4499988555908203
+[2025-12-29 02:49:44,786] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.44726085662841797
+[2025-12-29 02:49:45,191] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.40509533882141113
+[2025-12-29 02:49:45,192] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:3751] gather_len_batches: [1333]
+
  0%|                                                                                                                             | 0/3996 [00:00<?, ?it/s][2025-12-29 02:49:45,254] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:3751] Running evaluation step...
+[2025-12-29 02:49:46,094] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.42073726654052734
+[2025-12-29 02:49:46,520] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.4251554012298584
+[2025-12-29 02:49:46,891] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.37137746810913086
+[2025-12-29 02:49:47,274] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.38274621963500977
+[2025-12-29 02:49:47,274] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:3751] gather_len_batches: [72]
 
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.40it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:54,  1.59it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<01:00,  1.43it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:05,  1.30it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:06,  1.27it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:08,  1.21it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:06<01:07,  1.21it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:09,  1.16it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:08,  1.18it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:08,  1.15it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:06,  1.17it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:07,  1.14it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:05,  1.17it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:06,  1.12it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:13<01:03,  1.17it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:13<01:04,  1.14it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:02,  1.16it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:02,  1.13it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<01:00,  1.16it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<00:59,  1.15it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:58,  1.17it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:19<00:58,  1.15it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:19<00:56,  1.18it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:56,  1.15it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.18it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:54,  1.16it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:52,  1.18it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:52,  1.16it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:25<00:51,  1.18it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:25<00:51,  1.15it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.18it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:49,  1.15it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:47,  1.17it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:47,  1.16it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:45,  1.18it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:45,  1.16it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:31<00:44,  1.18it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:32<00:43,  1.16it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:42,  1.18it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:42,  1.16it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:40,  1.18it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:40,  1.15it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:39,  1.17it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:37<00:38,  1.16it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:38<00:37,  1.18it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.16it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:35,  1.18it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.16it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:33,  1.18it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:33,  1.16it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:43<00:32,  1.18it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:44<00:31,  1.16it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.18it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.16it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:28,  1.18it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.16it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.18it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:27,  1.12it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:50<00:26,  1.15it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:51<00:25,  1.13it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:24,  1.16it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:53<00:23,  1.14it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:22,  1.16it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:22,  1.13it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:20,  1.16it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:57<00:20,  1.12it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:57<00:19,  1.15it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:58<00:18,  1.14it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.16it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.15it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.17it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.14it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.18it/s]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:03<00:12,  1.15it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:04<00:11,  1.17it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:05<00:11,  1.16it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.17it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.15it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.17it/s]
-
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 81/90 [01:09<00:07,  1.15it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:09<00:06,  1.18it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:10<00:06,  1.13it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:11<00:05,  1.16it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.09it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.13it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.12it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:15<00:01,  1.15it/s]
-
 99%|████████████████████████████████████████████████████████████████��███████████████████████████████████████████████████▋ | 89/90 [01:16<00:00,  1.14it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.558133065700531, 'eval_runtime': 79.3297, 'eval_samples_per_second': 9.202, 'eval_steps_per_second': 2.307, 'eval_ppl': 1.7474, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.15, 'epoch': 0.6}
-
 20%|██████████████████████▏                                                                                        | 500/2499 [1:00:55<3:28:52,  6.27s/it]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
-
                                                                                                                                                           
 20%|██████████████████████                                                                                        | 501/2499 [1:01:01<17:23:40, 31.34s/it]
                                                                                                                                                           
{'loss': 0.5576, 'grad_norm': 0.16626505553722382, 'learning_rate': 0.00018147644911410403, 'ppl': 1.7465, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4281.06, 'total_tokens': 21995397, 'epoch': 0.6}
-
 20%|██████████████████████                                                                                        | 501/2499 [1:01:01<17:23:40, 31.34s/it]
 20%|██████████████████████                                                                                        | 502/2499 [1:01:08<13:12:52, 23.82s/it]
                                                                                                                                                           
{'loss': 0.5778, 'grad_norm': 0.15038293600082397, 'learning_rate': 0.00018140320341877294, 'ppl': 1.7821, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.09, 'tokens_per_second_per_gpu': 4543.93, 'total_tokens': 22023909, 'epoch': 0.6}
-
 20%|██████████████████████                                                                                        | 502/2499 [1:01:08<13:12:52, 23.82s/it]
 20%|██████████████████████▏                                                                                       | 503/2499 [1:01:14<10:17:47, 18.57s/it]
                                                                                                                                                           
{'loss': 0.5623, 'grad_norm': 0.17294828593730927, 'learning_rate': 0.000181329828037951, 'ppl': 1.7547, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4691.37, 'total_tokens': 22053532, 'epoch': 0.6}
-
 20%|██████████████████████▏                                                                                       | 503/2499 [1:01:14<10:17:47, 18.57s/it]
 20%|██████████████████████▍                                                                                        | 504/2499 [1:01:20<8:14:45, 14.88s/it]
                                                                                                                                                           
{'loss': 0.6039, 'grad_norm': 0.1541517972946167, 'learning_rate': 0.00018125632308853428, 'ppl': 1.8292, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4649.14, 'total_tokens': 22082656, 'epoch': 0.61}
-
 20%|██████████████████████▍                                                                                        | 504/2499 [1:01:20<8:14:45, 14.88s/it]
 20%|██████████████████████▍                                                                                        | 505/2499 [1:01:27<6:48:39, 12.30s/it]
                                                                                                                                                           
{'loss': 0.595, 'grad_norm': 0.1545095294713974, 'learning_rate': 0.00018118268868762546, 'ppl': 1.813, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4657.62, 'total_tokens': 22111841, 'epoch': 0.61}
-
 20%|██████████████████████▍                                                                                        | 505/2499 [1:01:27<6:48:39, 12.30s/it]
 20%|██████████████████████▍                                                                                        | 506/2499 [1:01:33<5:48:15, 10.48s/it]
                                                                                                                                                           
{'loss': 0.549, 'grad_norm': 0.16639526188373566, 'learning_rate': 0.0001811089249525333, 'ppl': 1.7315, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4463.92, 'total_tokens': 22139760, 'epoch': 0.61}
-
 20%|██████████████████████▍                                                                                        | 506/2499 [1:01:33<5:48:15, 10.48s/it]
 20%|██████████████████████▌                                                                                        | 507/2499 [1:01:39<5:05:46,  9.21s/it]
                                                                                                                                                           
{'loss': 0.5683, 'grad_norm': 0.16104137897491455, 'learning_rate': 0.00018103503200077263, 'ppl': 1.7653, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4327.85, 'total_tokens': 22166738, 'epoch': 0.61}
-
 20%|██████████████████████▌                                                                                        | 507/2499 [1:01:39<5:05:46,  9.21s/it]
 20%|██████████████████████▌                                                                                        | 508/2499 [1:01:45<4:36:23,  8.33s/it]
                                                                                                                                                           
{'loss': 0.5331, 'grad_norm': 0.15296021103858948, 'learning_rate': 0.00018096100995006423, 'ppl': 1.7042, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4610.98, 'total_tokens': 22195659, 'epoch': 0.61}
-
 20%|██████████████████████▌                                                                                        | 508/2499 [1:01:45<4:36:23,  8.33s/it]
 20%|██████████████████████▌                                                                                        | 509/2499 [1:01:52<4:16:06,  7.72s/it]
                                                                                                                                                           
{'loss': 0.5354, 'grad_norm': 0.1645633429288864, 'learning_rate': 0.0001808868589183345, 'ppl': 1.7081, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4609.55, 'total_tokens': 22224706, 'epoch': 0.61}
-
 20%|██████████████████████▌                                                                                        | 509/2499 [1:01:52<4:16:06,  7.72s/it]
 20%|██████████████████████▋                                                                                        | 510/2499 [1:01:58<4:01:43,  7.29s/it]
                                                                                                                                                           
{'loss': 0.5756, 'grad_norm': 0.15640902519226074, 'learning_rate': 0.00018081257902371524, 'ppl': 1.7782, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4596.26, 'total_tokens': 22253596, 'epoch': 0.61}
-
 20%|██████████████████████▋                                                                                        | 510/2499 [1:01:58<4:01:43,  7.29s/it]
 20%|██████████████████████▋                                                                                        | 511/2499 [1:02:04<3:51:21,  6.98s/it]
                                                                                                                                                           
{'loss': 0.5518, 'grad_norm': 0.1623287796974182, 'learning_rate': 0.00018073817038454364, 'ppl': 1.7364, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4549.9, 'total_tokens': 22282074, 'epoch': 0.61}
-
 20%|██████████████████████▋                                                                                        | 511/2499 [1:02:04<3:51:21,  6.98s/it]
 20%|██████████████████████▋                                                                                        | 512/2499 [1:02:10<3:44:01,  6.76s/it]
                                                                                                                                                           
{'loss': 0.6065, 'grad_norm': 0.162057563662529, 'learning_rate': 0.00018066363311936202, 'ppl': 1.834, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4456.85, 'total_tokens': 22309942, 'epoch': 0.61}
-
 20%|██████████████████████▋                                                                                        | 512/2499 [1:02:10<3:44:01,  6.76s/it]
 21%|██████████████████████▊                                                                                        | 513/2499 [1:02:17<3:39:09,  6.62s/it]
                                                                                                                                                           
{'loss': 0.5823, 'grad_norm': 0.153705894947052, 'learning_rate': 0.00018058896734691757, 'ppl': 1.7902, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4708.83, 'total_tokens': 22339534, 'epoch': 0.62}
-
 21%|██████████████████████▊                                                                                        | 513/2499 [1:02:17<3:39:09,  6.62s/it]
 21%|██████████████████████▊                                                                                        | 514/2499 [1:02:23<3:35:36,  6.52s/it]
                                                                                                                                                           
{'loss': 0.5846, 'grad_norm': 0.15547023713588715, 'learning_rate': 0.00018051417318616216, 'ppl': 1.7943, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4614.58, 'total_tokens': 22368473, 'epoch': 0.62}
-
 21%|██████████████████████▊                                                                                        | 514/2499 [1:02:23<3:35:36,  6.52s/it]
 21%|██████████████████████▉                                                                                        | 515/2499 [1:02:29<3:33:12,  6.45s/it]
                                                                                                                                                           
{'loss': 0.5752, 'grad_norm': 0.1760839819908142, 'learning_rate': 0.00018043925075625228, 'ppl': 1.7775, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4526.63, 'total_tokens': 22396916, 'epoch': 0.62}
-
 21%|██████████████████████▉                                                                                        | 515/2499 [1:02:29<3:33:12,  6.45s/it]
 21%|██████████████████████▉                                                                                        | 516/2499 [1:02:36<3:31:38,  6.40s/it]
                                                                                                                                                           
{'loss': 0.5958, 'grad_norm': 0.16130919754505157, 'learning_rate': 0.00018036420017654873, 'ppl': 1.8145, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4500.47, 'total_tokens': 22425258, 'epoch': 0.62}
-
 21%|██████████████████████▉                                                                                        | 516/2499 [1:02:36<3:31:38,  6.40s/it]
 21%|██████████████████████▉                                                                                        | 517/2499 [1:02:42<3:30:02,  6.36s/it]
                                                                                                                                                           
{'loss': 0.517, 'grad_norm': 0.16441930830478668, 'learning_rate': 0.0001802890215666165, 'ppl': 1.677, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4414.22, 'total_tokens': 22452848, 'epoch': 0.62}
-
 21%|██████████████████████▉                                                                                        | 517/2499 [1:02:42<3:30:02,  6.36s/it]
 21%|███████████████████████                                                                                        | 518/2499 [1:02:48<3:28:58,  6.33s/it]
                                                                                                                                                           
{'loss': 0.5451, 'grad_norm': 0.15389007329940796, 'learning_rate': 0.00018021371504622452, 'ppl': 1.7248, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4508.95, 'total_tokens': 22481068, 'epoch': 0.62}
-
 21%|███████████████████████                                                                                        | 518/2499 [1:02:48<3:28:58,  6.33s/it]
 21%|███████████████████████                                                                                        | 519/2499 [1:02:54<3:28:11,  6.31s/it]
                                                                                                                                                           
{'loss': 0.5338, 'grad_norm': 0.1578591912984848, 'learning_rate': 0.00018013828073534547, 'ppl': 1.7054, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4450.28, 'total_tokens': 22508921, 'epoch': 0.62}
-
 21%|███████████████████████                                                                                        | 519/2499 [1:02:54<3:28:11,  6.31s/it]
 21%|███████████████████████                                                                                        | 520/2499 [1:03:01<3:27:45,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5422, 'grad_norm': 0.1555330753326416, 'learning_rate': 0.00018006271875415565, 'ppl': 1.7198, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4617.04, 'total_tokens': 22537882, 'epoch': 0.62}
-
 21%|███████████████████████                                                                                        | 520/2499 [1:03:01<3:27:45,  6.30s/it]
 21%|███████████████████████▏                                                                                       | 521/2499 [1:03:07<3:27:18,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5466, 'grad_norm': 0.15180836617946625, 'learning_rate': 0.00017998702922303477, 'ppl': 1.7274, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4530.3, 'total_tokens': 22566249, 'epoch': 0.63}
-
 21%|███████████████████████▏                                                                                       | 521/2499 [1:03:07<3:27:18,  6.29s/it]
 21%|███████████████████████▏                                                                                       | 522/2499 [1:03:13<3:27:28,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5626, 'grad_norm': 0.1607424020767212, 'learning_rate': 0.00017991121226256573, 'ppl': 1.7552, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4533.77, 'total_tokens': 22594873, 'epoch': 0.63}
-
 21%|███████████████████████▏                                                                                       | 522/2499 [1:03:13<3:27:28,  6.30s/it]
 21%|███████████████████████▏                                                                                       | 523/2499 [1:03:20<3:27:17,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5573, 'grad_norm': 0.15108801424503326, 'learning_rate': 0.00017983526799353442, 'ppl': 1.746, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4619.24, 'total_tokens': 22623906, 'epoch': 0.63}
-
 21%|███████████████████████▏                                                                                       | 523/2499 [1:03:20<3:27:17,  6.29s/it]
 21%|███████████████████████▎                                                                                       | 524/2499 [1:03:26<3:26:46,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5722, 'grad_norm': 0.16330508887767792, 'learning_rate': 0.0001797591965369296, 'ppl': 1.7722, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4432.07, 'total_tokens': 22651604, 'epoch': 0.63}
-
 21%|███████████████████████▎                                                                                       | 524/2499 [1:03:26<3:26:46,  6.28s/it]
 21%|███████████████████████▎                                                                                       | 525/2499 [1:03:32<3:26:37,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5893, 'grad_norm': 0.1627909243106842, 'learning_rate': 0.00017968299801394258, 'ppl': 1.8027, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4696.21, 'total_tokens': 22681077, 'epoch': 0.63}
-
 21%|███████████████████████▎                                                                                       | 525/2499 [1:03:32<3:26:37,  6.28s/it]
 21%|███████████████████████▎                                                                                       | 526/2499 [1:03:38<3:26:37,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5747, 'grad_norm': 0.17663459479808807, 'learning_rate': 0.00017960667254596715, 'ppl': 1.7766, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4553.23, 'total_tokens': 22709704, 'epoch': 0.63}
-
 21%|███████████████████████▎                                                                                       | 526/2499 [1:03:38<3:26:37,  6.28s/it]
 21%|███████████████████████▍                                                                                       | 527/2499 [1:03:45<3:26:30,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5806, 'grad_norm': 0.163115993142128, 'learning_rate': 0.00017953022025459935, 'ppl': 1.7871, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4494.26, 'total_tokens': 22737926, 'epoch': 0.63}
-
 21%|███████████████████████▍                                                                                       | 527/2499 [1:03:45<3:26:30,  6.28s/it]
 21%|███████████████████████▍                                                                                       | 528/2499 [1:03:51<3:26:11,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6016, 'grad_norm': 0.16277560591697693, 'learning_rate': 0.00017945364126163723, 'ppl': 1.825, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4393.03, 'total_tokens': 22765427, 'epoch': 0.63}
-
 21%|███████████████████████▍                                                                                       | 528/2499 [1:03:51<3:26:11,  6.28s/it]
 21%|███████████████████████▍                                                                                       | 529/2499 [1:03:57<3:26:24,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5403, 'grad_norm': 0.15362772345542908, 'learning_rate': 0.00017937693568908074, 'ppl': 1.7165, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4542.41, 'total_tokens': 22794074, 'epoch': 0.64}
-
 21%|███████████████████████▍                                                                                       | 529/2499 [1:03:57<3:26:24,  6.29s/it]
 21%|███████████████████████▌                                                                                       | 530/2499 [1:04:04<3:26:25,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5604, 'grad_norm': 0.15853963792324066, 'learning_rate': 0.00017930010365913143, 'ppl': 1.7514, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4623.27, 'total_tokens': 22823175, 'epoch': 0.64}
-
 21%|███████████████████████▌                                                                                       | 530/2499 [1:04:04<3:26:25,  6.29s/it]
 21%|███████████████████████▌                                                                                       | 531/2499 [1:04:10<3:26:10,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5523, 'grad_norm': 0.1560271829366684, 'learning_rate': 0.00017922314529419237, 'ppl': 1.7372, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4598.04, 'total_tokens': 22852023, 'epoch': 0.64}
-
 21%|███████████████████████▌                                                                                       | 531/2499 [1:04:10<3:26:10,  6.29s/it]
 21%|███████████████████████▋                                                                                       | 532/2499 [1:04:16<3:26:03,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6234, 'grad_norm': 0.1705106794834137, 'learning_rate': 0.00017914606071686785, 'ppl': 1.8653, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4721.8, 'total_tokens': 22881681, 'epoch': 0.64}
-
 21%|███████████████████████▋                                                                                       | 532/2499 [1:04:16<3:26:03,  6.29s/it]
 21%|███████████████████████▋                                                                                       | 533/2499 [1:04:22<3:25:45,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5503, 'grad_norm': 0.15545393526554108, 'learning_rate': 0.0001790688500499632, 'ppl': 1.7338, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4532.04, 'total_tokens': 22910069, 'epoch': 0.64}
-
 21%|███████████████████████▋                                                                                       | 533/2499 [1:04:22<3:25:45,  6.28s/it]
 21%|███████████████████████▋                                                                                       | 534/2499 [1:04:29<3:25:34,  6.28s/it]
                                                                                                                                                           
{'loss': 0.578, 'grad_norm': 0.17963027954101562, 'learning_rate': 0.00017899151341648474, 'ppl': 1.7825, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4564.36, 'total_tokens': 22938685, 'epoch': 0.64}
-
 21%|███████████████████████▋                                                                                       | 534/2499 [1:04:29<3:25:34,  6.28s/it]
 21%|███████████████████████▊                                                                                       | 535/2499 [1:04:35<3:25:39,  6.28s/it]
                                                                                                                                                           
{'loss': 0.59, 'grad_norm': 0.1561817079782486, 'learning_rate': 0.00017891405093963938, 'ppl': 1.804, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4383.98, 'total_tokens': 22966271, 'epoch': 0.64}
-
 21%|███████████████████████▊                                                                                       | 535/2499 [1:04:35<3:25:39,  6.28s/it]
 21%|███████████████████████▊                                                                                       | 536/2499 [1:04:41<3:25:52,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5487, 'grad_norm': 0.174911230802536, 'learning_rate': 0.00017883646274283454, 'ppl': 1.731, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4498.07, 'total_tokens': 22994666, 'epoch': 0.64}
-
 21%|███████████████████████▊                                                                                       | 536/2499 [1:04:41<3:25:52,  6.29s/it]
 21%|███████████████████████▊                                                                                       | 537/2499 [1:04:48<3:25:43,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6081, 'grad_norm': 0.16251260042190552, 'learning_rate': 0.00017875874894967794, 'ppl': 1.8369, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4616.4, 'total_tokens': 23023673, 'epoch': 0.64}
-
 21%|███████████████████████▊                                                                                       | 537/2499 [1:04:48<3:25:43,  6.29s/it]
 22%|███████████████████████▉                                                                                       | 538/2499 [1:04:54<3:25:12,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5542, 'grad_norm': 0.15792444348335266, 'learning_rate': 0.00017868090968397736, 'ppl': 1.7405, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4314.14, 'total_tokens': 23050620, 'epoch': 0.65}
-
 22%|███████████████████████▉                                                                                       | 538/2499 [1:04:54<3:25:12,  6.28s/it]
 22%|███████████████████████▉                                                                                       | 539/2499 [1:05:00<3:25:08,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5239, 'grad_norm': 0.1523372232913971, 'learning_rate': 0.00017860294506974055, 'ppl': 1.6886, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4679.92, 'total_tokens': 23080008, 'epoch': 0.65}
-
 22%|███████████████████████▉                                                                                       | 539/2499 [1:05:00<3:25:08,  6.28s/it]
 22%|███████████████████████▉                                                                                       | 540/2499 [1:05:06<3:24:49,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5764, 'grad_norm': 0.1880243867635727, 'learning_rate': 0.00017852485523117478, 'ppl': 1.7796, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4379.76, 'total_tokens': 23107406, 'epoch': 0.65}
-
 22%|███████████████████████▉                                                                                       | 540/2499 [1:05:06<3:24:49,  6.27s/it]
 22%|████████████████████████                                                                                       | 541/2499 [1:05:13<3:24:27,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5599, 'grad_norm': 0.16542772948741913, 'learning_rate': 0.00017844664029268708, 'ppl': 1.7505, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4374.44, 'total_tokens': 23134723, 'epoch': 0.65}
-
 22%|████████████████████████                                                                                       | 541/2499 [1:05:13<3:24:27,  6.27s/it]
 22%|████████████████████████                                                                                       | 542/2499 [1:05:19<3:24:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5629, 'grad_norm': 0.1714017689228058, 'learning_rate': 0.0001783683003788835, 'ppl': 1.7558, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4514.61, 'total_tokens': 23163201, 'epoch': 0.65}
-
 22%|████████████████████████                                                                                       | 542/2499 [1:05:19<3:24:47,  6.28s/it]
 22%|████████████████████████                                                                                       | 543/2499 [1:05:25<3:24:46,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5781, 'grad_norm': 0.1670805811882019, 'learning_rate': 0.00017828983561456941, 'ppl': 1.7826, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4516.32, 'total_tokens': 23191579, 'epoch': 0.65}
-
 22%|████████████████████████                                                                                       | 543/2499 [1:05:25<3:24:46,  6.28s/it]
 22%|████████████████████████▏                                                                                      | 544/2499 [1:05:31<3:24:30,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5766, 'grad_norm': 0.17114055156707764, 'learning_rate': 0.00017821124612474903, 'ppl': 1.78, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4499.03, 'total_tokens': 23219753, 'epoch': 0.65}
-
 22%|████████████████████████▏                                                                                      | 544/2499 [1:05:31<3:24:30,  6.28s/it]
 22%|████████████████████████▏                                                                                      | 545/2499 [1:05:38<3:24:07,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5891, 'grad_norm': 0.15948644280433655, 'learning_rate': 0.00017813253203462516, 'ppl': 1.8024, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4505.12, 'total_tokens': 23247888, 'epoch': 0.65}
-
 22%|████████████████████████▏                                                                                      | 545/2499 [1:05:38<3:24:07,  6.27s/it]
 22%|████████████████████████▎                                                                                      | 546/2499 [1:05:44<3:24:05,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5609, 'grad_norm': 0.15668705105781555, 'learning_rate': 0.00017805369346959925, 'ppl': 1.7522, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4661.63, 'total_tokens': 23277134, 'epoch': 0.66}
-
 22%|████████████████████████▎                                                                                      | 546/2499 [1:05:44<3:24:05,  6.27s/it]
 22%|████████████████████████▎                                                                                      | 547/2499 [1:05:50<3:24:03,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5408, 'grad_norm': 0.1964712142944336, 'learning_rate': 0.00017797473055527097, 'ppl': 1.7174, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4663.52, 'total_tokens': 23306393, 'epoch': 0.66}
-
 22%|████████████████████████▎                                                                                      | 547/2499 [1:05:50<3:24:03,  6.27s/it]
 22%|████████████████████████▎                                                                                      | 548/2499 [1:05:56<3:23:52,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5684, 'grad_norm': 0.15374480187892914, 'learning_rate': 0.00017789564341743816, 'ppl': 1.7654, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4344.09, 'total_tokens': 23333599, 'epoch': 0.66}
-
 22%|████████████████████████▎                                                                                      | 548/2499 [1:05:56<3:23:52,  6.27s/it]
 22%|████████████████████████▍                                                                                      | 549/2499 [1:06:03<3:24:11,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5538, 'grad_norm': 0.15823286771774292, 'learning_rate': 0.00017781643218209653, 'ppl': 1.7399, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4599.04, 'total_tokens': 23362610, 'epoch': 0.66}
-
 22%|████████████████████████▍                                                                                      | 549/2499 [1:06:03<3:24:11,  6.28s/it]
 22%|████████████████████████▍                                                                                      | 550/2499 [1:06:09<3:24:00,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5801, 'grad_norm': 0.16051071882247925, 'learning_rate': 0.00017773709697543945, 'ppl': 1.7862, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4632.44, 'total_tokens': 23391660, 'epoch': 0.66}
-
 22%|████████████████████████▍                                                                                      | 550/2499 [1:06:09<3:24:00,  6.28s/it]
 22%|████████████████████████▍                                                                                      | 551/2499 [1:06:15<3:23:39,  6.27s/it]
                                                                                                                                                           
{'loss': 0.547, 'grad_norm': 0.14237217605113983, 'learning_rate': 0.00017765763792385787, 'ppl': 1.7281, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4429.86, 'total_tokens': 23419362, 'epoch': 0.66}
-
 22%|████████████████████████▍                                                                                      | 551/2499 [1:06:15<3:23:39,  6.27s/it]
 22%|████████████████████████▌                                                                                      | 552/2499 [1:06:22<3:23:37,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6078, 'grad_norm': 0.16329942643642426, 'learning_rate': 0.00017757805515394002, 'ppl': 1.8364, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4671.82, 'total_tokens': 23448687, 'epoch': 0.66}
-
 22%|████████████████████████▌                                                                                      | 552/2499 [1:06:22<3:23:37,  6.27s/it]
 22%|████████████████████████▌                                                                                      | 553/2499 [1:06:28<3:23:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5276, 'grad_norm': 0.1630595177412033, 'learning_rate': 0.00017749834879247117, 'ppl': 1.6949, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4539.17, 'total_tokens': 23477121, 'epoch': 0.66}
-
 22%|████████████████████████▌                                                                                      | 553/2499 [1:06:28<3:23:26,  6.27s/it]
 22%|████████████████████████▌                                                                                      | 554/2499 [1:06:34<3:23:12,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5946, 'grad_norm': 0.16599886119365692, 'learning_rate': 0.0001774185189664335, 'ppl': 1.8123, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4517.7, 'total_tokens': 23505395, 'epoch': 0.67}
-
 22%|████████████████████████▌                                                                                      | 554/2499 [1:06:34<3:23:12,  6.27s/it]
 22%|████████████████████████▋                                                                                      | 555/2499 [1:06:40<3:23:20,  6.28s/it]
                                                                                                                                                           
{'loss': 0.557, 'grad_norm': 0.16257159411907196, 'learning_rate': 0.000177338565803006, 'ppl': 1.7454, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4267.3, 'total_tokens': 23532239, 'epoch': 0.67}
-
 22%|████████████████████████▋                                                                                      | 555/2499 [1:06:40<3:23:20,  6.28s/it]
 22%|████████████████████████▋                                                                                      | 556/2499 [1:06:47<3:23:19,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5678, 'grad_norm': 0.16005942225456238, 'learning_rate': 0.00017725848942956406, 'ppl': 1.7644, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4605.1, 'total_tokens': 23561162, 'epoch': 0.67}
-
 22%|████████████████████████▋                                                                                      | 556/2499 [1:06:47<3:23:19,  6.28s/it]
 22%|████████████████████████▋                                                                                      | 557/2499 [1:06:53<3:23:18,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5562, 'grad_norm': 0.14812323451042175, 'learning_rate': 0.00017717828997367927, 'ppl': 1.744, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4736.45, 'total_tokens': 23590929, 'epoch': 0.67}
-
 22%|████████████████████████▋                                                                                      | 557/2499 [1:06:53<3:23:18,  6.28s/it]
 22%|████████████████████████▊                                                                                      | 558/2499 [1:06:59<3:23:04,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6006, 'grad_norm': 0.170892596244812, 'learning_rate': 0.00017709796756311947, 'ppl': 1.8232, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4511.82, 'total_tokens': 23619199, 'epoch': 0.67}
-
 22%|████████████████████████▊                                                                                      | 558/2499 [1:06:59<3:23:04,  6.28s/it]
 22%|████████████████████████▊                                                                                      | 559/2499 [1:07:06<3:22:51,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5399, 'grad_norm': 0.15551912784576416, 'learning_rate': 0.0001770175223258483, 'ppl': 1.7158, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4474.15, 'total_tokens': 23647226, 'epoch': 0.67}
-
 22%|████████████████████████▊                                                                                      | 559/2499 [1:07:06<3:22:51,  6.27s/it]
 22%|████████████████████████▊                                                                                      | 560/2499 [1:07:12<3:22:43,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5606, 'grad_norm': 0.1615440845489502, 'learning_rate': 0.00017693695439002514, 'ppl': 1.7517, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4531.95, 'total_tokens': 23675631, 'epoch': 0.67}
-
 22%|████████████████████████▊                                                                                      | 560/2499 [1:07:12<3:22:43,  6.27s/it]
 22%|████████████████████████▉                                                                                      | 561/2499 [1:07:18<3:22:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5892, 'grad_norm': 0.15482234954833984, 'learning_rate': 0.00017685626388400468, 'ppl': 1.8025, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4402.47, 'total_tokens': 23703313, 'epoch': 0.67}
-
 22%|████████████████████████▉                                                                                      | 561/2499 [1:07:18<3:22:47,  6.28s/it]
 22%|████████████████████████▉                                                                                      | 562/2499 [1:07:24<3:22:56,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5729, 'grad_norm': 0.15045323967933655, 'learning_rate': 0.00017677545093633713, 'ppl': 1.7734, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4530.74, 'total_tokens': 23731860, 'epoch': 0.67}
-
 22%|████████████████████████▉                                                                                      | 562/2499 [1:07:24<3:22:56,  6.29s/it]
 23%|█████████████████████████                                                                                      | 563/2499 [1:07:31<3:22:38,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5675, 'grad_norm': 0.14530088007450104, 'learning_rate': 0.00017669451567576752, 'ppl': 1.7639, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4520.18, 'total_tokens': 23760164, 'epoch': 0.68}
-
 23%|█████████████████████████                                                                                      | 563/2499 [1:07:31<3:22:38,  6.28s/it]
 23%|█████████████████████████                                                                                      | 564/2499 [1:07:37<3:22:16,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5739, 'grad_norm': 0.16555465757846832, 'learning_rate': 0.00017661345823123589, 'ppl': 1.7752, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4475.26, 'total_tokens': 23788137, 'epoch': 0.68}
-
 23%|█████████████████████████                                                                                      | 564/2499 [1:07:37<3:22:16,  6.27s/it]
 23%|█████████████████████████                                                                                      | 565/2499 [1:07:43<3:21:57,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5543, 'grad_norm': 0.16551339626312256, 'learning_rate': 0.0001765322787318769, 'ppl': 1.7407, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4461.73, 'total_tokens': 23816015, 'epoch': 0.68}
-
 23%|█████████████████████████                                                                                      | 565/2499 [1:07:43<3:21:57,  6.27s/it]
 23%|█████████████████████████▏                                                                                     | 566/2499 [1:07:49<3:21:45,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5611, 'grad_norm': 0.14666950702667236, 'learning_rate': 0.00017645097730701966, 'ppl': 1.7526, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4523.64, 'total_tokens': 23844296, 'epoch': 0.68}
-
 23%|█████████████████████████▏                                                                                     | 566/2499 [1:07:49<3:21:45,  6.26s/it]
 23%|█████████████████████████▏                                                                                     | 567/2499 [1:07:56<3:21:35,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5476, 'grad_norm': 0.15764540433883667, 'learning_rate': 0.00017636955408618753, 'ppl': 1.7291, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4582.64, 'total_tokens': 23872958, 'epoch': 0.68}
-
 23%|█████████████████████████▏                                                                                     | 567/2499 [1:07:56<3:21:35,  6.26s/it]
 23%|█████████████████████████▏                                                                                     | 568/2499 [1:08:02<3:21:44,  6.27s/it]
                                                                                                                                                           
{'loss': 0.599, 'grad_norm': 0.17149189114570618, 'learning_rate': 0.0001762880091990978, 'ppl': 1.8203, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4476.09, 'total_tokens': 23901085, 'epoch': 0.68}
-
 23%|█████████████████████████▏                                                                                     | 568/2499 [1:08:02<3:21:44,  6.27s/it]
 23%|█████████████████████████▎                                                                                     | 569/2499 [1:08:08<3:21:28,  6.26s/it]
                                                                                                                                                           
{'loss': 0.569, 'grad_norm': 0.16149137914180756, 'learning_rate': 0.00017620634277566176, 'ppl': 1.7665, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4416.6, 'total_tokens': 23928683, 'epoch': 0.68}
-
 23%|█████████████████████████▎                                                                                     | 569/2499 [1:08:08<3:21:28,  6.26s/it]
 23%|████████████████████���████▎                                                                                     | 570/2499 [1:08:14<3:21:17,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5524, 'grad_norm': 0.15485823154449463, 'learning_rate': 0.0001761245549459843, 'ppl': 1.7374, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4413.88, 'total_tokens': 23956279, 'epoch': 0.68}
-
 23%|█████████████████████████▎                                                                                     | 570/2499 [1:08:14<3:21:17,  6.26s/it]
 23%|█████████████████████████▎                                                                                     | 571/2499 [1:08:21<3:21:19,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5226, 'grad_norm': 0.1360737830400467, 'learning_rate': 0.0001760426458403635, 'ppl': 1.6864, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4749.99, 'total_tokens': 23986075, 'epoch': 0.69}
-
 23%|█████████████████████████▎                                                                                     | 571/2499 [1:08:21<3:21:19,  6.27s/it]
 23%|█████████████████████████▍                                                                                     | 572/2499 [1:08:27<3:21:08,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5439, 'grad_norm': 0.14969180524349213, 'learning_rate': 0.00017596061558929096, 'ppl': 1.7227, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4661.33, 'total_tokens': 24015230, 'epoch': 0.69}
-
 23%|█████████████████████████▍                                                                                     | 572/2499 [1:08:27<3:21:08,  6.26s/it]
 23%|█████████████████████████▍                                                                                     | 573/2499 [1:08:33<3:21:04,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5821, 'grad_norm': 0.15753692388534546, 'learning_rate': 0.0001758784643234511, 'ppl': 1.7898, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4587.04, 'total_tokens': 24043965, 'epoch': 0.69}
-
 23%|█████████████████████████▍                                                                                     | 573/2499 [1:08:33<3:21:04,  6.26s/it]
 23%|█████████████████████████▍                                                                                     | 574/2499 [1:08:40<3:21:18,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5036, 'grad_norm': 0.14932945370674133, 'learning_rate': 0.00017579619217372115, 'ppl': 1.6547, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4529.01, 'total_tokens': 24072482, 'epoch': 0.69}
-
 23%|█████████████████████████▍                                                                                     | 574/2499 [1:08:40<3:21:18,  6.27s/it]
 23%|█████████████████████████▌                                                                                     | 575/2499 [1:08:46<3:21:38,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5811, 'grad_norm': 0.15429732203483582, 'learning_rate': 0.0001757137992711709, 'ppl': 1.788, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4593.01, 'total_tokens': 24101493, 'epoch': 0.69}
-
 23%|█████████████████████████▌                                                                                     | 575/2499 [1:08:46<3:21:38,  6.29s/it]
 23%|█████████████████████████▌                                                                                     | 576/2499 [1:08:52<3:21:23,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5526, 'grad_norm': 0.1494465470314026, 'learning_rate': 0.00017563128574706263, 'ppl': 1.7378, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4648.78, 'total_tokens': 24130640, 'epoch': 0.69}
-
 23%|█████████████████████████▌                                                                                     | 576/2499 [1:08:52<3:21:23,  6.28s/it]
 23%|█████████████████████████▋                                                                                     | 577/2499 [1:08:58<3:21:09,  6.28s/it]
                                                                                                                                                           
{'loss': 0.583, 'grad_norm': 0.16360746324062347, 'learning_rate': 0.00017554865173285075, 'ppl': 1.7914, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4551.95, 'total_tokens': 24159164, 'epoch': 0.69}
-
 23%|█████████████████████████▋                                                                                     | 577/2499 [1:08:58<3:21:09,  6.28s/it]
 23%|█████████████████████████▋                                                                                     | 578/2499 [1:09:05<3:20:43,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5222, 'grad_norm': 0.15340928733348846, 'learning_rate': 0.00017546589736018147, 'ppl': 1.6857, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4430.58, 'total_tokens': 24186825, 'epoch': 0.69}
-
 23%|█████████████████████████▋                                                                                     | 578/2499 [1:09:05<3:20:43,  6.27s/it]
 23%|█████████████████████████▋                                                                                     | 579/2499 [1:09:11<3:20:19,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5551, 'grad_norm': 0.15968795120716095, 'learning_rate': 0.0001753830227608929, 'ppl': 1.7421, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4416.18, 'total_tokens': 24214365, 'epoch': 0.7}
-
 23%|█████████████████████████▋                                                                                     | 579/2499 [1:09:11<3:20:19,  6.26s/it]
 23%|█████████████████████████▊                                                                                     | 580/2499 [1:09:17<3:20:53,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5779, 'grad_norm': 0.16069112718105316, 'learning_rate': 0.0001753000280670147, 'ppl': 1.7823, 'memory/max_active (GiB)': 17.85, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4409.81, 'total_tokens': 24242271, 'epoch': 0.7}
-
 23%|█████████████████████████▊                                                                                     | 580/2499 [1:09:17<3:20:53,  6.28s/it]
 23%|█████████████████████████▊                                                                                     | 581/2499 [1:09:24<3:22:34,  6.34s/it]
                                                                                                                                                           
{'loss': 0.5631, 'grad_norm': 0.156789168715477, 'learning_rate': 0.00017521691341076774, 'ppl': 1.7561, 'memory/max_active (GiB)': 17.43, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4547.78, 'total_tokens': 24271665, 'epoch': 0.7}
-
 23%|█████████████████████████▊                                                                                     | 581/2499 [1:09:24<3:22:34,  6.34s/it]
 23%|█████████████████████████▊                                                                                     | 582/2499 [1:09:30<3:23:19,  6.36s/it]
                                                                                                                                                           
{'loss': 0.5613, 'grad_norm': 0.17415142059326172, 'learning_rate': 0.00017513367892456406, 'ppl': 1.7529, 'memory/max_active (GiB)': 13.45, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4414.67, 'total_tokens': 24300005, 'epoch': 0.7}
-
 23%|█████████████████████████▊                                                                                     | 582/2499 [1:09:30<3:23:19,  6.36s/it]
 23%|█████████████████████████▉                                                                                     | 583/2499 [1:09:37<3:23:19,  6.37s/it]
                                                                                                                                                           
{'loss': 0.549, 'grad_norm': 0.15976881980895996, 'learning_rate': 0.00017505032474100674, 'ppl': 1.7315, 'memory/max_active (GiB)': 17.32, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4433.8, 'total_tokens': 24328253, 'epoch': 0.7}
-
 23%|█████████████████████████▉                                                                                     | 583/2499 [1:09:37<3:23:19,  6.37s/it]
 23%|█████████████████████████▉                                                                                     | 584/2499 [1:09:43<3:23:03,  6.36s/it]
                                                                                                                                                           
{'loss': 0.5844, 'grad_norm': 0.15100276470184326, 'learning_rate': 0.00017496685099288928, 'ppl': 1.7939, 'memory/max_active (GiB)': 17.64, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4370.46, 'total_tokens': 24355986, 'epoch': 0.7}
-
 23%|█████████████████████████▉                                                                                     | 584/2499 [1:09:43<3:23:03,  6.36s/it]
 23%|█████████████████████████▉                                                                                     | 585/2499 [1:09:49<3:22:55,  6.36s/it]
                                                                                                                                                           
{'loss': 0.5733, 'grad_norm': 0.16124789416790009, 'learning_rate': 0.00017488325781319587, 'ppl': 1.7741, 'memory/max_active (GiB)': 17.46, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4415.51, 'total_tokens': 24384052, 'epoch': 0.7}
-
 23%|█████████████████████████▉                                                                                     | 585/2499 [1:09:49<3:22:55,  6.36s/it]
 23%|██████████████████████████                                                                                     | 586/2499 [1:09:56<3:22:59,  6.37s/it]
                                                                                                                                                           
{'loss': 0.6033, 'grad_norm': 0.15764689445495605, 'learning_rate': 0.00017479954533510087, 'ppl': 1.8281, 'memory/max_active (GiB)': 17.43, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4608.31, 'total_tokens': 24413426, 'epoch': 0.7}
-
 23%|██████████████████████████                                                                                     | 586/2499 [1:09:56<3:22:59,  6.37s/it]
 23%|██████████████████████████                                                                                     | 587/2499 [1:10:02<3:23:26,  6.38s/it]
                                                                                                                                                           
{'loss': 0.5235, 'grad_norm': 0.14312417805194855, 'learning_rate': 0.00017471571369196881, 'ppl': 1.6879, 'memory/max_active (GiB)': 17.28, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4291.88, 'total_tokens': 24440985, 'epoch': 0.7}
-
 23%|██████████████████████████                                                                                     | 587/2499 [1:10:02<3:23:26,  6.38s/it]
 24%|██████████████████████████                                                                                     | 588/2499 [1:10:09<3:24:24,  6.42s/it]
                                                                                                                                                           
{'loss': 0.5914, 'grad_norm': 0.17255191504955292, 'learning_rate': 0.00017463176301735396, 'ppl': 1.8065, 'memory/max_active (GiB)': 13.95, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4468.74, 'total_tokens': 24469984, 'epoch': 0.71}
-
 24%|██████████████████████████                                                                                     | 588/2499 [1:10:09<3:24:24,  6.42s/it]
 24%|██████████████████████████▏                                                                                    | 589/2499 [1:10:15<3:23:17,  6.39s/it]
                                                                                                                                                           
{'loss': 0.5315, 'grad_norm': 0.1607024073600769, 'learning_rate': 0.0001745476934450002, 'ppl': 1.7015, 'memory/max_active (GiB)': 17.28, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4434.69, 'total_tokens': 24497954, 'epoch': 0.71}
-
 24%|██████████████████████████▏                                                                                    | 589/2499 [1:10:15<3:23:17,  6.39s/it]
 24%|██████████████████████████▏                                                                                    | 590/2499 [1:10:21<3:22:00,  6.35s/it]
                                                                                                                                                           
{'loss': 0.5643, 'grad_norm': 0.15177768468856812, 'learning_rate': 0.00017446350510884093, 'ppl': 1.7582, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4475.09, 'total_tokens': 24525965, 'epoch': 0.71}
-
 24%|██████████████████████████▏                                                                                    | 590/2499 [1:10:21<3:22:00,  6.35s/it]
 24%|██████████████████████████▎                                                                                    | 591/2499 [1:10:27<3:21:11,  6.33s/it]
                                                                                                                                                           
{'loss': 0.5545, 'grad_norm': 0.15134449303150177, 'learning_rate': 0.0001743791981429987, 'ppl': 1.7411, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4533.18, 'total_tokens': 24554401, 'epoch': 0.71}
-
 24%|██████████████████████████▎                                                                                    | 591/2499 [1:10:27<3:21:11,  6.33s/it]
 24%|██████████████████████████▎                                                                                    | 592/2499 [1:10:34<3:20:29,  6.31s/it]
                                                                                                                                                           
{'loss': 0.5247, 'grad_norm': 0.1504749059677124, 'learning_rate': 0.00017429477268178503, 'ppl': 1.69, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4499.53, 'total_tokens': 24582574, 'epoch': 0.71}
-
 24%|██████████████████████████▎                                                                                    | 592/2499 [1:10:34<3:20:29,  6.31s/it]
 24%|██████████████████████████▎                                                                                    | 593/2499 [1:10:40<3:19:49,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5007, 'grad_norm': 0.14617754518985748, 'learning_rate': 0.0001742102288597003, 'ppl': 1.6499, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4500.48, 'total_tokens': 24610693, 'epoch': 0.71}
-
 24%|██████████████████████████▎                                                                                    | 593/2499 [1:10:40<3:19:49,  6.29s/it]
 24%|██████████████████████████▍                                                                                    | 594/2499 [1:10:46<3:19:47,  6.29s/it]
                                                                                                                                                           
{'loss': 0.6026, 'grad_norm': 0.15605376660823822, 'learning_rate': 0.0001741255668114333, 'ppl': 1.8269, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4591.72, 'total_tokens': 24639598, 'epoch': 0.71}
-
 24%|██████████████████████████▍                                                                                    | 594/2499 [1:10:46<3:19:47,  6.29s/it]
 24%|██████████████████████████▍                                                                                    | 595/2499 [1:10:52<3:19:26,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5504, 'grad_norm': 0.17370787262916565, 'learning_rate': 0.00017404078667186142, 'ppl': 1.7339, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4447.33, 'total_tokens': 24667454, 'epoch': 0.71}
-
 24%|██████████████████████████▍                                                                                    | 595/2499 [1:10:52<3:19:26,  6.29s/it]
 24%|██████████████████████████▍                                                                                    | 596/2499 [1:10:59<3:18:57,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5451, 'grad_norm': 0.1539432853460312, 'learning_rate': 0.0001739558885760499, 'ppl': 1.7248, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4360.5, 'total_tokens': 24694675, 'epoch': 0.72}
-
 24%|██████████████████████████▍                                                                                    | 596/2499 [1:10:59<3:18:57,  6.27s/it]
 24%|██████████████████████████▌                                                                                    | 597/2499 [1:11:05<3:18:45,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5394, 'grad_norm': 0.1412273794412613, 'learning_rate': 0.00017387087265925208, 'ppl': 1.715, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4564.48, 'total_tokens': 24723246, 'epoch': 0.72}
-
 24%|██████████████████████████▌                                                                                    | 597/2499 [1:11:05<3:18:45,  6.27s/it]
 24%|██████████████████████████▌                                                                                    | 598/2499 [1:11:11<3:18:32,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5957, 'grad_norm': 0.16358250379562378, 'learning_rate': 0.00017378573905690896, 'ppl': 1.8143, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4510.4, 'total_tokens': 24751464, 'epoch': 0.72}
-
 24%|██████████████████████████▌                                                                                    | 598/2499 [1:11:11<3:18:32,  6.27s/it]
 24%|██████████████████████████▌                                                                                    | 599/2499 [1:11:17<3:18:16,  6.26s/it]
                                                                                                                                                           
{'loss': 0.4988, 'grad_norm': 0.15359358489513397, 'learning_rate': 0.00017370048790464902, 'ppl': 1.6467, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4345.71, 'total_tokens': 24778614, 'epoch': 0.72}
-
 24%|██████████████████████████▌                                                                                    | 599/2499 [1:11:17<3:18:16,  6.26s/it]
 24%|██████████████████████████▋                                                                                    | 600/2499 [1:11:24<3:18:31,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5896, 'grad_norm': 0.1726973056793213, 'learning_rate': 0.00017361511933828801, 'ppl': 1.8033, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.17, 'tokens_per_second_per_gpu': 4595.75, 'total_tokens': 24807549, 'epoch': 0.72}
-
 24%|██████████████████████████▋                                                                                    | 600/2499 [1:11:24<3:18:31,  6.27s/it][2025-12-28 12:17:00,390] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 12:17:02,934] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 1.2528719902038574
-[2025-12-28 12:17:04,233] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 1.2981853485107422
-[2025-12-28 12:17:05,552] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 1.319084644317627
-[2025-12-28 12:17:06,805] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 1.2518165111541748
-[2025-12-28 12:17:06,805] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
+
  0%|                                                                                                                               | 0/72 [00:00<?, ?it/s]
+
  3%|███▎                                                                                                                   | 2/72 [00:02<01:19,  1.14s/it]
+
  4%|████▉                                                                                                                  | 3/72 [00:04<01:50,  1.61s/it]
+
  6%|██████▌                                                                                                                | 4/72 [00:06<02:06,  1.86s/it]
+
  7%|████████▎                                                                                                              | 5/72 [00:09<02:14,  2.00s/it]
+
  8%|█████████▉                                                                                                             | 6/72 [00:11<02:18,  2.10s/it]
+
 10%|███████████▌                                                                                                           | 7/72 [00:13<02:19,  2.15s/it]
+
 11%|█████████████▏                                                                                                         | 8/72 [00:15<02:20,  2.19s/it]
+
 12%|██████████████▉                                                                                                        | 9/72 [00:18<02:19,  2.22s/it]
+
 14%|████████████████▍                                                                                                     | 10/72 [00:20<02:18,  2.23s/it]
+
 15%|██████████████████                                                                                                    | 11/72 [00:22<02:18,  2.27s/it]
+
 17%|███████████████████▋                                                                                                  | 12/72 [00:25<02:18,  2.30s/it]
+
 18%|█████████████████████▎                                                                                                | 13/72 [00:27<02:16,  2.31s/it]
+
 19%|██████████████████████▉                                                                                               | 14/72 [00:29<02:13,  2.30s/it]
+
 21%|████████████████████████▌                                                                                             | 15/72 [00:32<02:11,  2.30s/it]
+
 22%|██████████████████████████▏                                                                                           | 16/72 [00:34<02:08,  2.30s/it]
+
 24%|███████████████████████████▊                                                                                          | 17/72 [00:36<02:06,  2.30s/it]
+
 25%|█████████████████████████████▌                                                                                        | 18/72 [00:38<02:03,  2.29s/it]
+
 26%|███████████████████████████████▏                                                                                      | 19/72 [00:41<02:01,  2.29s/it]
+
 28%|████████████████████████████████▊                                                                                     | 20/72 [00:43<01:58,  2.29s/it]
+
 29%|██████████████████████████████████▍                                                                                   | 21/72 [00:45<01:56,  2.28s/it]
+
 31%|████████████████████████████████████                                                                                  | 22/72 [00:48<01:53,  2.28s/it]
+
 32%|█████████████████████████████████████▋                                                                                | 23/72 [00:50<01:51,  2.28s/it]
+
 33%|███████████████████████████████████████▎                                                                              | 24/72 [00:52<01:49,  2.28s/it]
+
 35%|████████████████████████████████████████▉                                                                             | 25/72 [00:54<01:47,  2.29s/it]
+
 36%|██████████████████████████████████████████▌                                                                           | 26/72 [00:57<01:51,  2.43s/it]
+
 38%|████████████████████████████████████████████▎                                                                         | 27/72 [00:59<01:47,  2.38s/it]
+
 39%|█████████████████████████████████████████████▉                                                                        | 28/72 [01:02<01:42,  2.34s/it]
+
 40%|███████████████████████████████████████████████▌                                                                      | 29/72 [01:04<01:39,  2.30s/it]
+
 42%|█████████████████████████████████████████████████▏                                                                    | 30/72 [01:06<01:35,  2.28s/it]
+
 43%|██████████████████████████████████████████████████▊                                                                   | 31/72 [01:08<01:33,  2.28s/it]
+
 44%|████████████████████████████████████████████████████▍                                                                 | 32/72 [01:11<01:30,  2.27s/it]
+
 46%|██████████████████████████████████████████████████████                                                                | 33/72 [01:13<01:28,  2.26s/it]
+
 47%|███████████████████████████████████████████████████████▋                                                              | 34/72 [01:15<01:25,  2.26s/it]
+
 49%|█████████████████████████████████████████████████████████▎                                                            | 35/72 [01:17<01:23,  2.26s/it]
+
 50%|███████████████████████████████████████████████████████████                                                           | 36/72 [01:20<01:21,  2.26s/it]
+
 51%|█████████████████████████████████████████████���██████████████▋                                                         | 37/72 [01:22<01:18,  2.25s/it]
+
 53%|██████████████████████████████████████████████████████████████▎                                                       | 38/72 [01:24<01:16,  2.24s/it]
+
 54%|███████████████████████████████████████████████████████████████▉                                                      | 39/72 [01:26<01:14,  2.24s/it]
+
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 40/72 [01:29<01:11,  2.25s/it]
+
 57%|███████████████████████████████████████████████████████████████████▏                                                  | 41/72 [01:31<01:09,  2.25s/it]
+
 58%|████████████████████████████████████████████████████████████████████▊                                                 | 42/72 [01:33<01:07,  2.25s/it]
+
 60%|██████████████████████████████████████████████████████████████████████▍                                               | 43/72 [01:35<01:05,  2.25s/it]
+
 61%|████████████████████████████████████████████████████████████████████████                                              | 44/72 [01:38<01:08,  2.45s/it]
+
 62%|█████████████████████████████████████████████████████████████████████████▊                                            | 45/72 [01:41<01:05,  2.43s/it]
+
 64%|███████████████████████████████████████████████████████████████████████████▍                                          | 46/72 [01:43<01:01,  2.38s/it]
+
 65%|█████████████████████████████████████████████████████████████████████████████                                         | 47/72 [01:45<00:58,  2.34s/it]
+
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 48/72 [01:48<00:55,  2.32s/it]
+
 68%|████████████████████████████████████████████████████████████████████████████████▎                                     | 49/72 [01:50<00:53,  2.31s/it]
+
 69%|█████████████████████████████████████████████████████████████████████████████████▉                                    | 50/72 [01:52<00:50,  2.30s/it]
+
 71%|███████████████████████████████████████████████████████████████████████████████████▌                                  | 51/72 [01:54<00:48,  2.29s/it]
+
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 52/72 [01:57<00:45,  2.29s/it]
+
 74%|██████████████████████████████████████████████████████████████████████████████████████▊                               | 53/72 [01:59<00:43,  2.28s/it]
+
 75%|████████████████████████████████████████████████████████████████████████████████████████▌                             | 54/72 [02:01<00:41,  2.28s/it]
+
 76%|██████████████████████████████████████████████████████████████████████████████████████████▏                           | 55/72 [02:03<00:38,  2.27s/it]
+
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 56/72 [02:06<00:36,  2.26s/it]
+
 79%|█████████████████████████████████████████████████████████████████████████████████████████████▍                        | 57/72 [02:08<00:33,  2.26s/it]
+
 81%|███████████████████████████████████████████████████████████████████████████████████████████████                       | 58/72 [02:10<00:31,  2.27s/it]
+
 82%|████████████████████████████████████████████████████████████████████████████████████████████████▋                     | 59/72 [02:14<00:34,  2.68s/it]
+
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 60/72 [02:16<00:30,  2.56s/it]
+
 85%|███████████████████████████████████████████████████████████████████████████████████████████████████▉                  | 61/72 [02:18<00:27,  2.48s/it]
+
 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌                | 62/72 [02:21<00:24,  2.42s/it]
+
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎              | 63/72 [02:23<00:21,  2.37s/it]
+
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 64/72 [02:25<00:18,  2.33s/it]
+
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 65/72 [02:27<00:16,  2.31s/it]
+
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 66/72 [02:30<00:13,  2.30s/it]
+
 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 67/72 [02:32<00:11,  2.30s/it]
+
 94%|█████████████████████████��█████████████████████████████████████████████████████████████████████████████████████▍      | 68/72 [02:34<00:09,  2.30s/it]
+
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████     | 69/72 [02:37<00:06,  2.29s/it]
+
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 70/72 [02:39<00:04,  2.29s/it]
+
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 71/72 [02:41<00:02,  2.29s/it]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:43<00:00,  2.30s/it]
                                                                                                                                                           
+
                                                                                                                                                           
{'eval_loss': 1.6887853145599365, 'eval_runtime': 167.3526, 'eval_samples_per_second': 4.362, 'eval_steps_per_second': 0.872, 'eval_ppl': 5.4129, 'memory/max_active (GiB)': 18.94, 'memory/max_allocated (GiB)': 18.94, 'memory/device_reserved (GiB)': 139.12, 'epoch': 0}
+
  0%|                                                                                                                             | 0/3996 [02:49<?, ?it/s]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:44<00:00,  2.30s/it]
+
                                                                                                                                                           
  0%|                                                                                                                | 1/3996 [02:55<195:12:22, 175.91s/it]
  0%|                                                                                                                  | 2/3996 [03:00<83:06:37, 74.91s/it]
  0%|                                                                                                                  | 3/3996 [03:04<47:17:31, 42.64s/it]
  0%|                                                                                                                  | 4/3996 [03:08<30:28:11, 27.48s/it]
  0%|▏                                                                                                                 | 5/3996 [03:13<21:23:10, 19.29s/it]
  0%|▏                                                                                                                 | 6/3996 [03:17<15:42:55, 14.18s/it]
  0%|▏                                                                                                                 | 7/3996 [03:21<12:06:13, 10.92s/it]
  0%|▏                                                                                                                  | 8/3996 [03:26<9:44:18,  8.79s/it]
  0%|▎                                                                                                                  | 9/3996 [03:30<8:09:49,  7.37s/it]
  0%|▎                                                                                                                 | 10/3996 [03:34<7:05:16,  6.40s/it]
  0%|▎                                                                                                                 | 11/3996 [03:39<6:29:39,  5.87s/it]
  0%|▎                                                                                                                 | 12/3996 [03:43<6:07:23,  5.53s/it]
  0%|▎                                                                                                                 | 13/3996 [03:48<5:41:57,  5.15s/it]
  0%|▍                                                                                                                 | 14/3996 [03:52<5:23:28,  4.87s/it]
  0%|▍                                                                                                                 | 15/3996 [03:56<5:10:29,  4.68s/it]
  0%|▍                                                                                                                 | 16/3996 [04:00<5:01:50,  4.55s/it]
  0%|▍                                                                                                                 | 17/3996 [04:05<4:55:19,  4.45s/it]
  0%|▌                                                                                                                 | 18/3996 [04:09<4:51:02,  4.39s/it]
  0%|▌                                                                                                                 | 19/3996 [04:14<4:58:50,  4.51s/it]
  1%|▌                                                                                                                 | 20/3996 [04:18<4:53:38,  4.43s/it]
  1%|▌                                                                                                                 | 21/3996 [04:22<4:49:34,  4.37s/it]
  1%|▋                                                                                                                 | 22/3996 [04:26<4:46:39,  4.33s/it]
  1%|▋                                                                                                                 | 23/3996 [04:31<4:44:50,  4.30s/it]
  1%|▋                                                                                                                 | 24/3996 [04:35<4:43:31,  4.28s/it]
  1%|▋                                                                                                                 | 25/3996 [04:39<4:42:32,  4.27s/it]
                                                                                                                                                           
{'loss': 1.6848, 'grad_norm': 1.415561556816101, 'learning_rate': 4.8e-05, 'ppl': 5.3914, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.11, 'tokens_per_second_per_gpu': 16277.76, 'total_tokens': 1723633, 'epoch': 0.02}
+
  1%|▋                                                                                                                 | 25/3996 [04:39<4:42:32,  4.27s/it]
  1%|▋                                                                                                                 | 26/3996 [04:44<4:52:31,  4.42s/it]
  1%|▊                                                                                                                 | 27/3996 [04:48<4:48:56,  4.37s/it]
  1%|▊                                                                                                                 | 28/3996 [04:52<4:45:57,  4.32s/it]
  1%|▊                                                                                                                 | 29/3996 [04:57<4:44:01,  4.30s/it]
  1%|▊                                                                                                                 | 30/3996 [05:01<4:42:52,  4.28s/it]
  1%|▉                                                                                                                 | 31/3996 [05:05<4:41:56,  4.27s/it]
  1%|▉                                                                                                                 | 32/3996 [05:09<4:41:16,  4.26s/it]
  1%|▉                                                                                                                 | 33/3996 [05:14<4:51:21,  4.41s/it]
  1%|▉                                                                                                                 | 34/3996 [05:18<4:47:45,  4.36s/it]
  1%|▉                                                                                                                 | 35/3996 [05:23<4:45:08,  4.32s/it]
  1%|█                                                                                                                 | 36/3996 [05:27<4:42:59,  4.29s/it]
  1%|█                                                                                                                 | 37/3996 [05:31<4:41:30,  4.27s/it]
  1%|█                                                                                                                 | 38/3996 [05:35<4:40:27,  4.25s/it]
  1%|█                                                                                                                 | 39/3996 [05:39<4:40:02,  4.25s/it]
  1%|█▏                                                                                                                | 40/3996 [05:44<4:50:09,  4.40s/it]
  1%|█▏                                                                                                                | 41/3996 [05:48<4:46:35,  4.35s/it]
  1%|█▏                                                                                                                | 42/3996 [05:53<4:43:43,  4.31s/it]
  1%|█▏                                                                                                                | 43/3996 [05:57<4:41:52,  4.28s/it]
  1%|█▎                                                                                                                | 44/3996 [06:01<4:40:47,  4.26s/it]
  1%|█▎                                                                                                                | 45/3996 [06:05<4:39:54,  4.25s/it]
  1%|█▎                                                                                                                | 46/3996 [06:10<4:40:37,  4.26s/it]
  1%|█▎                                                                                                                | 47/3996 [06:15<4:56:40,  4.51s/it]
  1%|█▎                                                                                                                | 48/3996 [06:19<4:50:53,  4.42s/it]
  1%|█▍                                                                                                                | 49/3996 [06:23<4:46:26,  4.35s/it]
  1%|█▍                                                                                                                | 50/3996 [06:27<4:43:19,  4.31s/it]
                                                                                                                                                           
{'loss': 0.9839, 'grad_norm': 0.33179354667663574, 'learning_rate': 9.8e-05, 'ppl': 2.6749, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4303.21, 'total_tokens': 2175386, 'epoch': 0.04}
+
  1%|█▍                                                                                                                | 50/3996 [06:27<4:43:19,  4.31s/it]
  1%|█▍                                                                                                                | 51/3996 [06:32<4:41:34,  4.28s/it]
  1%|█▍                                                                                                                | 52/3996 [06:36<4:40:11,  4.26s/it]
  1%|█▌                                                                                                                | 53/3996 [06:40<4:39:22,  4.25s/it]
  1%|█▌                                                                                                                | 54/3996 [06:45<4:49:56,  4.41s/it]
  1%|█▌                                                                                                                | 55/3996 [06:49<4:45:49,  4.35s/it]
  1%|█▌                                                                                                                | 56/3996 [06:53<4:42:56,  4.31s/it]
  1%|█▋                                                                                                                | 57/3996 [06:57<4:40:54,  4.28s/it]
  1%|█▋                                                                                                                | 58/3996 [07:02<4:39:43,  4.26s/it]
  1%|█▋                                                                                                                | 59/3996 [07:06<4:38:47,  4.25s/it]
  2%|█▋                                                                                                                | 60/3996 [07:10<4:38:14,  4.24s/it]
  2%|█▋                                                                                                                | 61/3996 [07:15<4:48:03,  4.39s/it]
  2%|█▊                                                                                                                | 62/3996 [07:19<4:44:49,  4.34s/it]
  2%|█▊                                                                                                                | 63/3996 [07:23<4:42:22,  4.31s/it]
  2%|█▊                                                                                                                | 64/3996 [07:27<4:40:20,  4.28s/it]
  2%|█▊                                                                                                                | 65/3996 [07:32<4:42:49,  4.32s/it]
  2%|█▉                                                                                                                | 66/3996 [07:36<4:40:56,  4.29s/it]
  2%|█▉                                                                                                                | 67/3996 [07:40<4:39:50,  4.27s/it]
  2%|█▉                                                                                                                | 68/3996 [07:45<4:49:20,  4.42s/it]
  2%|█▉                                                                                                                | 69/3996 [07:49<4:45:27,  4.36s/it]
  2%|█▉                                                                                                                | 70/3996 [07:53<4:42:09,  4.31s/it]
  2%|██                                                                                                                | 71/3996 [07:58<4:39:53,  4.28s/it]
  2%|██                                                                                                                | 72/3996 [08:02<4:38:38,  4.26s/it]
  2%|██                                                                                                                | 73/3996 [08:06<4:37:42,  4.25s/it]
  2%|██                                                                                                                | 74/3996 [08:10<4:37:29,  4.25s/it]
  2%|██▏                                                                                                               | 75/3996 [08:15<4:47:19,  4.40s/it]
                                                                                                                                                           
{'loss': 0.8002, 'grad_norm': 0.17453454434871674, 'learning_rate': 0.000148, 'ppl': 2.226, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3776.03, 'total_tokens': 2623712, 'epoch': 0.06}
+
  2%|██▏                                                                                                               | 75/3996 [08:15<4:47:19,  4.40s/it]
  2%|██▏                                                                                                               | 76/3996 [08:19<4:43:59,  4.35s/it]
  2%|██▏                                                                                                               | 77/3996 [08:24<4:41:37,  4.31s/it]
  2%|██▏                                                                                                               | 78/3996 [08:28<4:39:35,  4.28s/it]
  2%|██▎                                                                                                               | 79/3996 [08:32<4:38:38,  4.27s/it]
  2%|██▎                                                                                                               | 80/3996 [08:36<4:37:43,  4.26s/it]
  2%|██▎                                                                                                               | 81/3996 [08:40<4:37:20,  4.25s/it]
  2%|██▎                                                                                                               | 82/3996 [08:46<4:54:33,  4.52s/it]
  2%|██▎                                                                                                               | 83/3996 [08:50<4:48:41,  4.43s/it]
  2%|██▍                                                                                                               | 84/3996 [08:54<4:44:41,  4.37s/it]
  2%|██▍                                                                                                               | 85/3996 [08:58<4:41:24,  4.32s/it]
  2%|██▍                                                                                                               | 86/3996 [09:02<4:39:38,  4.29s/it]
  2%|██▍                                                                                                               | 87/3996 [09:07<4:38:17,  4.27s/it]
  2%|██▌                                                                                                               | 88/3996 [09:11<4:39:34,  4.29s/it]
  2%|██▌                                                                                                               | 89/3996 [09:16<4:48:40,  4.43s/it]
  2%|██▌                                                                                                               | 90/3996 [09:20<4:44:42,  4.37s/it]
  2%|██▌                                                                                                               | 91/3996 [09:24<4:42:08,  4.34s/it]
  2%|██▌                                                                                                               | 92/3996 [09:29<4:39:42,  4.30s/it]
  2%|██▋                                                                                                               | 93/3996 [09:33<4:38:01,  4.27s/it]
  2%|██▋                                                                                                               | 94/3996 [09:37<4:37:00,  4.26s/it]
  2%|██▋                                                                                                               | 95/3996 [09:41<4:36:15,  4.25s/it]
  2%|██▋                                                                                                               | 96/3996 [09:46<4:46:17,  4.40s/it]
  2%|██▊                                                                                                               | 97/3996 [09:50<4:42:25,  4.35s/it]
  2%|██▊                                                                                                               | 98/3996 [09:54<4:39:55,  4.31s/it]
  2%|██▊                                                                                                               | 99/3996 [09:59<4:37:46,  4.28s/it]
  3%|██▊                                                                                                              | 100/3996 [10:03<4:36:38,  4.26s/it]
                                                                                                                                                           
{'loss': 0.7218, 'grad_norm': 0.19318008422851562, 'learning_rate': 0.00019800000000000002, 'ppl': 2.0581, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4252.49, 'total_tokens': 3072519, 'epoch': 0.08}
+
  3%|██▊                                                                                                              | 100/3996 [10:03<4:36:38,  4.26s/it]
  3%|██▊                                                                                                              | 101/3996 [10:07<4:35:55,  4.25s/it]
  3%|██▉                                                                                                              | 102/3996 [10:11<4:35:32,  4.25s/it]
  3%|██▉                                                                                                              | 103/3996 [10:16<4:45:41,  4.40s/it]
  3%|██▉                                                                                                              | 104/3996 [10:20<4:42:04,  4.35s/it]
  3%|██▉                                                                                                              | 105/3996 [10:24<4:39:28,  4.31s/it]
  3%|██▉                                                                                                              | 106/3996 [10:29<4:37:26,  4.28s/it]
  3%|███                                                                                                              | 107/3996 [10:33<4:36:09,  4.26s/it]
  3%|███                                                                                                              | 108/3996 [10:37<4:34:54,  4.24s/it]
  3%|███                                                                                                              | 109/3996 [10:41<4:34:20,  4.23s/it]
  3%|███                                                                                                              | 110/3996 [10:46<4:44:18,  4.39s/it]
  3%|███▏                                                                                                             | 111/3996 [10:50<4:41:05,  4.34s/it]
  3%|███▏                                                                                                             | 112/3996 [10:55<4:38:28,  4.30s/it]
  3%|███▏                                                                                                             | 113/3996 [10:59<4:36:31,  4.27s/it]
  3%|███▏                                                                                                             | 114/3996 [11:03<4:35:21,  4.26s/it]
  3%|███▎                                                                                                             | 115/3996 [11:07<4:34:23,  4.24s/it]
  3%|███▎                                                                                                             | 116/3996 [11:11<4:33:49,  4.23s/it]
  3%|███▎                                                                                                             | 117/3996 [11:16<4:43:29,  4.39s/it]
  3%|███▎                                                                                                             | 118/3996 [11:21<4:47:24,  4.45s/it]
  3%|███▎                                                                                                             | 119/3996 [11:25<4:42:25,  4.37s/it]
  3%|███▍                                                                                                             | 120/3996 [11:29<4:39:07,  4.32s/it]
  3%|███▍                                                                                                             | 121/3996 [11:33<4:37:00,  4.29s/it]
  3%|███▍                                                                                                             | 122/3996 [11:38<4:35:18,  4.26s/it]
  3%|███▍                                                                                                             | 123/3996 [11:42<4:34:23,  4.25s/it]
  3%|███▌                                                                                                             | 124/3996 [11:46<4:43:49,  4.40s/it]
  3%|███▌                                                                                                             | 125/3996 [11:51<4:39:51,  4.34s/it]
                                                                                                                                                           
{'loss': 0.6759, 'grad_norm': 0.18435686826705933, 'learning_rate': 0.00019998127418269004, 'ppl': 1.9658, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4303.31, 'total_tokens': 3523983, 'epoch': 0.09}
+
  3%|███▌                                                                                                             | 125/3996 [11:51<4:39:51,  4.34s/it]
  3%|███▌                                                                                                             | 126/3996 [11:55<4:37:09,  4.30s/it]
  3%|███▌                                                                                                             | 127/3996 [11:59<4:34:51,  4.26s/it]
  3%|███▌                                                                                                             | 128/3996 [12:03<4:33:58,  4.25s/it]
  3%|███▋                                                                                                             | 129/3996 [12:07<4:32:59,  4.24s/it]
  3%|███▋                                                                                                             | 130/3996 [12:12<4:32:32,  4.23s/it]
  3%|███▋                                                                                                             | 131/3996 [12:16<4:42:23,  4.38s/it]
  3%|███▋                                                                                                             | 132/3996 [12:21<4:38:56,  4.33s/it]
  3%|███▊                                                                                                             | 133/3996 [12:25<4:36:21,  4.29s/it]
  3%|███▊                                                                                                             | 134/3996 [12:29<4:34:32,  4.27s/it]
  3%|███▊                                                                                                             | 135/3996 [12:33<4:33:25,  4.25s/it]
  3%|███▊                                                                                                             | 136/3996 [12:37<4:32:35,  4.24s/it]
  3%|███▊                                                                                                             | 137/3996 [12:42<4:31:44,  4.23s/it]
  3%|███▉                                                                                                             | 138/3996 [12:46<4:41:38,  4.38s/it]
  3%|███▉                                                                                                             | 139/3996 [12:51<4:38:06,  4.33s/it]
  4%|███▉                                                                                                             | 140/3996 [12:55<4:35:33,  4.29s/it]
  4%|███▉                                                                                                             | 141/3996 [12:59<4:33:47,  4.26s/it]
  4%|████                                                                                                             | 142/3996 [13:03<4:32:44,  4.25s/it]
  4%|████                                                                                                             | 143/3996 [13:07<4:31:48,  4.23s/it]
  4%|████                                                                                                             | 144/3996 [13:12<4:31:19,  4.23s/it]
  4%|████                                                                                                             | 145/3996 [13:16<4:40:57,  4.38s/it]
  4%|████▏                                                                                                            | 146/3996 [13:21<4:37:32,  4.33s/it]
  4%|████▏                                                                                                            | 147/3996 [13:25<4:35:15,  4.29s/it]
  4%|████▏                                                                                                            | 148/3996 [13:29<4:33:18,  4.26s/it]
  4%|████▏                                                                                                            | 149/3996 [13:33<4:32:24,  4.25s/it]
  4%|████▏                                                                                                            | 150/3996 [13:37<4:31:47,  4.24s/it]
                                                                                                                                                           
{'loss': 0.6703, 'grad_norm': 0.19870473444461823, 'learning_rate': 0.00019992195096972548, 'ppl': 1.9548, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4260.86, 'total_tokens': 3973452, 'epoch': 0.11}
+
  4%|████▏                                                                                                            | 150/3996 [13:37<4:31:47,  4.24s/it]
  4%|████▎                                                                                                            | 151/3996 [13:42<4:31:06,  4.23s/it]
  4%|████▎                                                                                                            | 152/3996 [13:46<4:40:51,  4.38s/it]
  4%|████▎                                                                                                            | 153/3996 [13:51<4:37:32,  4.33s/it]
  4%|████▎                                                                                                            | 154/3996 [13:55<4:35:48,  4.31s/it]
  4%|████▍                                                                                                            | 155/3996 [13:59<4:33:54,  4.28s/it]
  4%|████▍                                                                                                            | 156/3996 [14:03<4:32:45,  4.26s/it]
  4%|████▍                                                                                                            | 157/3996 [14:07<4:31:33,  4.24s/it]
  4%|████▍                                                                                                            | 158/3996 [14:12<4:31:00,  4.24s/it]
  4%|████▍                                                                                                            | 159/3996 [14:16<4:40:39,  4.39s/it]
  4%|████▌                                                                                                            | 160/3996 [14:21<4:37:18,  4.34s/it]
  4%|████▌                                                                                                            | 161/3996 [14:25<4:34:43,  4.30s/it]
  4%|████▌                                                                                                            | 162/3996 [14:29<4:33:08,  4.27s/it]
  4%|████▌                                                                                                            | 163/3996 [14:33<4:31:50,  4.26s/it]
  4%|████▋                                                                                                            | 164/3996 [14:37<4:31:02,  4.24s/it]
  4%|████▋                                                                                                            | 165/3996 [14:42<4:30:32,  4.24s/it]
  4%|████▋                                                                                                            | 166/3996 [14:46<4:40:20,  4.39s/it]
  4%|████▋                                                                                                            | 167/3996 [14:51<4:36:50,  4.34s/it]
  4%|████▊                                                                                                            | 168/3996 [14:55<4:34:14,  4.30s/it]
  4%|████▊                                                                                                            | 169/3996 [14:59<4:32:40,  4.27s/it]
  4%|████▊                                                                                                            | 170/3996 [15:03<4:31:39,  4.26s/it]
  4%|████▊                                                                                                            | 171/3996 [15:08<4:30:26,  4.24s/it]
  4%|████▊                                                                                                            | 172/3996 [15:12<4:30:27,  4.24s/it]
  4%|████▉                                                                                                            | 173/3996 [15:17<4:41:49,  4.42s/it]
  4%|████▉                                                                                                            | 174/3996 [15:21<4:38:06,  4.37s/it]
  4%|████▉                                                                                                            | 175/3996 [15:25<4:34:53,  4.32s/it]
                                                                                                                                                           
{'loss': 0.6381, 'grad_norm': 0.20499658584594727, 'learning_rate': 0.0001998220219574743, 'ppl': 1.8929, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4288.64, 'total_tokens': 4423763, 'epoch': 0.13}
+
  4%|████▉                                                                                                            | 175/3996 [15:25<4:34:53,  4.32s/it]
  4%|████▉                                                                                                            | 176/3996 [15:29<4:32:56,  4.29s/it]
  4%|█████                                                                                                            | 177/3996 [15:33<4:31:34,  4.27s/it]
  4%|█████                                                                                                            | 178/3996 [15:38<4:30:44,  4.25s/it]
  4%|█████                                                                                                            | 179/3996 [15:42<4:29:57,  4.24s/it]
  5%|█████                                                                                                            | 180/3996 [15:47<4:39:34,  4.40s/it]
  5%|█████                                                                                                            | 181/3996 [15:51<4:36:17,  4.35s/it]
  5%|█████▏                                                                                                           | 182/3996 [15:55<4:33:32,  4.30s/it]
  5%|█████▏                                                                                                           | 183/3996 [15:59<4:31:42,  4.28s/it]
  5%|█████▏                                                                                                           | 184/3996 [16:04<4:30:55,  4.26s/it]
  5%|█████▏                                                                                                           | 185/3996 [16:08<4:30:05,  4.25s/it]
  5%|█████▎                                                                                                           | 186/3996 [16:12<4:29:25,  4.24s/it]
  5%|█████▎                                                                                                           | 187/3996 [16:17<4:39:02,  4.40s/it]
  5%|█████▎                                                                                                           | 188/3996 [16:21<4:35:58,  4.35s/it]
  5%|█████▎                                                                                                           | 189/3996 [16:25<4:33:25,  4.31s/it]
  5%|█████▎                                                                                                           | 190/3996 [16:29<4:31:33,  4.28s/it]
  5%|█████▍                                                                                                           | 191/3996 [16:34<4:30:28,  4.27s/it]
  5%|█████▍                                                                                                           | 192/3996 [16:38<4:29:49,  4.26s/it]
  5%|█████▍                                                                                                           | 193/3996 [16:42<4:29:07,  4.25s/it]
  5%|█████▍                                                                                                           | 194/3996 [16:47<4:38:42,  4.40s/it]
  5%|█████▌                                                                                                           | 195/3996 [16:51<4:35:16,  4.35s/it]
  5%|█████▌                                                                                                           | 196/3996 [16:55<4:32:31,  4.30s/it]
  5%|█████▌                                                                                                           | 197/3996 [16:59<4:30:32,  4.27s/it]
  5%|█████▌                                                                                                           | 198/3996 [17:04<4:29:36,  4.26s/it]
  5%|█████▋                                                                                                           | 199/3996 [17:08<4:28:56,  4.25s/it]
  5%|█████▋                                                                                                           | 200/3996 [17:12<4:28:27,  4.24s/it]
                                                                                                                                                           
{'loss': 0.6383, 'grad_norm': 0.18934418261051178, 'learning_rate': 0.00019968152775460537, 'ppl': 1.8933, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4244.79, 'total_tokens': 4872365, 'epoch': 0.15}
+
  5%|█████▋                                                                                                           | 200/3996 [17:12<4:28:27,  4.24s/it]
  5%|█████▋                                                                                                           | 201/3996 [17:17<4:37:53,  4.39s/it]
  5%|█████▋                                                                                                           | 202/3996 [17:21<4:34:51,  4.35s/it]
  5%|█████▋                                                                                                           | 203/3996 [17:25<4:32:14,  4.31s/it]
  5%|█████▊                                                                                                           | 204/3996 [17:30<4:30:10,  4.28s/it]
  5%|█████▊                                                                                                           | 205/3996 [17:34<4:29:15,  4.26s/it]
  5%|█████▊                                                                                                           | 206/3996 [17:38<4:28:24,  4.25s/it]
  5%|█████▊                                                                                                           | 207/3996 [17:42<4:27:32,  4.24s/it]
  5%|█████▉                                                                                                           | 208/3996 [17:47<4:45:05,  4.52s/it]
  5%|█████▉                                                                                                           | 209/3996 [17:52<4:39:14,  4.42s/it]
  5%|█████▉                                                                                                           | 210/3996 [17:56<4:34:57,  4.36s/it]
  5%|█████▉                                                                                                           | 211/3996 [18:00<4:32:02,  4.31s/it]
  5%|█████▉                                                                                                           | 212/3996 [18:04<4:30:07,  4.28s/it]
  5%|██████                                                                                                           | 213/3996 [18:08<4:28:39,  4.26s/it]
  5%|██████                                                                                                           | 214/3996 [18:13<4:27:39,  4.25s/it]
  5%|██████                                                                                                           | 215/3996 [18:17<4:37:10,  4.40s/it]
  5%|██████                                                                                                           | 216/3996 [18:22<4:33:42,  4.34s/it]
  5%|██████▏                                                                                                          | 217/3996 [18:26<4:30:57,  4.30s/it]
  5%|██████▏                                                                                                          | 218/3996 [18:30<4:29:09,  4.27s/it]
  5%|██████▏                                                                                                          | 219/3996 [18:34<4:27:56,  4.26s/it]
  6%|██████▏                                                                                                          | 220/3996 [18:38<4:26:51,  4.24s/it]
  6%|██████▏                                                                                                          | 221/3996 [18:43<4:26:29,  4.24s/it]
  6%|██████▎                                                                                                          | 222/3996 [18:47<4:36:42,  4.40s/it]
  6%|██████▎                                                                                                          | 223/3996 [18:52<4:33:17,  4.35s/it]
  6%|██████▎                                                                                                          | 224/3996 [18:56<4:30:36,  4.30s/it]
  6%|██████▎                                                                                                          | 225/3996 [19:00<4:28:39,  4.27s/it]
                                                                                                                                                           
{'loss': 0.6347, 'grad_norm': 0.1827855408191681, 'learning_rate': 0.00019950052545447352, 'ppl': 1.8865, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4252.71, 'total_tokens': 5319322, 'epoch': 0.17}
+
  6%|██████▎                                                                                                          | 225/3996 [19:00<4:28:39,  4.27s/it]
  6%|██████▍                                                                                                          | 226/3996 [19:04<4:27:44,  4.26s/it]
  6%|██████▍                                                                                                          | 227/3996 [19:09<4:26:48,  4.25s/it]
  6%|██████▍                                                                                                          | 228/3996 [19:13<4:26:18,  4.24s/it]
  6%|██████▍                                                                                                          | 229/3996 [19:18<4:36:07,  4.40s/it]
  6%|██████▌                                                                                                          | 230/3996 [19:22<4:32:59,  4.35s/it]
  6%|██████▌                                                                                                          | 231/3996 [19:26<4:30:34,  4.31s/it]
  6%|██████▌                                                                                                          | 232/3996 [19:30<4:28:25,  4.28s/it]
  6%|██████▌                                                                                                          | 233/3996 [19:34<4:27:01,  4.26s/it]
  6%|██████▌                                                                                                          | 234/3996 [19:39<4:26:15,  4.25s/it]
  6%|██████▋                                                                                                          | 235/3996 [19:43<4:25:41,  4.24s/it]
  6%|██████▋                                                                                                          | 236/3996 [19:48<4:35:15,  4.39s/it]
  6%|██████▋                                                                                                          | 237/3996 [19:52<4:32:09,  4.34s/it]
  6%|██████▋                                                                                                          | 238/3996 [19:56<4:29:22,  4.30s/it]
  6%|██████▊                                                                                                          | 239/3996 [20:00<4:27:37,  4.27s/it]
  6%|██████▊                                                                                                          | 240/3996 [20:04<4:26:40,  4.26s/it]
  6%|██████▊                                                                                                          | 241/3996 [20:09<4:26:01,  4.25s/it]
  6%|██████▊                                                                                                          | 242/3996 [20:13<4:25:12,  4.24s/it]
  6%|██████▊                                                                                                          | 243/3996 [20:18<4:34:57,  4.40s/it]
  6%|██████▉                                                                                                          | 244/3996 [20:22<4:39:40,  4.47s/it]
  6%|██████▉                                                                                                          | 245/3996 [20:27<4:34:45,  4.40s/it]
  6%|██████▉                                                                                                          | 246/3996 [20:31<4:31:16,  4.34s/it]
  6%|██████▉                                                                                                          | 247/3996 [20:35<4:28:48,  4.30s/it]
  6%|███████                                                                                                          | 248/3996 [20:39<4:27:20,  4.28s/it]
  6%|███████                                                                                                          | 249/3996 [20:43<4:25:51,  4.26s/it]
  6%|███████                                                                                                          | 250/3996 [20:48<4:35:18,  4.41s/it]
                                                                                                                                                           
{'loss': 0.6392, 'grad_norm': 0.16483066976070404, 'learning_rate': 0.00019927908861191827, 'ppl': 1.895, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3772.0, 'total_tokens': 5768644, 'epoch': 0.19}
+
  6%|███████                                                                                                          | 250/3996 [20:48<4:35:18,  4.41s/it]
  6%|███████                                                                                                          | 251/3996 [20:52<4:31:39,  4.35s/it]
  6%|███████▏                                                                                                         | 252/3996 [20:57<4:29:05,  4.31s/it]
  6%|███████▏                                                                                                         | 253/3996 [21:01<4:26:53,  4.28s/it]
  6%|███████▏                                                                                                         | 254/3996 [21:05<4:25:58,  4.26s/it]
  6%|███████▏                                                                                                         | 255/3996 [21:09<4:25:23,  4.26s/it]
  6%|███████▏                                                                                                         | 256/3996 [21:13<4:24:51,  4.25s/it]
  6%|███████▎                                                                                                         | 257/3996 [21:18<4:34:27,  4.40s/it]
  6%|███████▎                                                                                                         | 258/3996 [21:23<4:31:25,  4.36s/it]
  6%|███████▎                                                                                                         | 259/3996 [21:27<4:28:53,  4.32s/it]
  7%|███████▎                                                                                                         | 260/3996 [21:31<4:26:55,  4.29s/it]
  7%|███████▍                                                                                                         | 261/3996 [21:35<4:26:02,  4.27s/it]
  7%|███████▍                                                                                                         | 262/3996 [21:39<4:25:00,  4.26s/it]
  7%|███████▍                                                                                                         | 263/3996 [21:44<4:24:14,  4.25s/it]
  7%|███████▍                                                                                                         | 264/3996 [21:48<4:33:49,  4.40s/it]
  7%|███████▍                                                                                                         | 265/3996 [21:53<4:30:28,  4.35s/it]
  7%|███████▌                                                                                                         | 266/3996 [21:57<4:27:47,  4.31s/it]
  7%|███████▌                                                                                                         | 267/3996 [22:01<4:26:01,  4.28s/it]
  7%|███████▌                                                                                                         | 268/3996 [22:05<4:25:24,  4.27s/it]
  7%|███████▌                                                                                                         | 269/3996 [22:10<4:24:39,  4.26s/it]
  7%|███████▋                                                                                                         | 270/3996 [22:14<4:23:48,  4.25s/it]
  7%|███████▋                                                                                                         | 271/3996 [22:19<4:33:10,  4.40s/it]
  7%|███████▋                                                                                                         | 272/3996 [22:23<4:30:02,  4.35s/it]
  7%|███████▋                                                                                                         | 273/3996 [22:27<4:27:28,  4.31s/it]
  7%|███████▋                                                                                                         | 274/3996 [22:31<4:25:45,  4.28s/it]
  7%|███████▊                                                                                                         | 275/3996 [22:35<4:24:34,  4.27s/it]
                                                                                                                                                           
{'loss': 0.614, 'grad_norm': 0.17186357080936432, 'learning_rate': 0.00019901730721337302, 'ppl': 1.8478, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4281.82, 'total_tokens': 6220751, 'epoch': 0.21}
+
  7%|███████▊                                                                                                         | 275/3996 [22:35<4:24:34,  4.27s/it]
  7%|███████▊                                                                                                         | 276/3996 [22:40<4:23:38,  4.25s/it]
  7%|███████▊                                                                                                         | 277/3996 [22:44<4:22:46,  4.24s/it]
  7%|███████▊                                                                                                         | 278/3996 [22:49<4:32:26,  4.40s/it]
  7%|███████▉                                                                                                         | 279/3996 [22:53<4:29:06,  4.34s/it]
  7%|███████▉                                                                                                         | 280/3996 [22:57<4:26:29,  4.30s/it]
  7%|███████▉                                                                                                         | 281/3996 [23:01<4:24:48,  4.28s/it]
  7%|███████▉                                                                                                         | 282/3996 [23:05<4:23:45,  4.26s/it]
  7%|████████                                                                                                         | 283/3996 [23:10<4:23:10,  4.25s/it]
  7%|████████                                                                                                         | 284/3996 [23:14<4:22:31,  4.24s/it]
  7%|████████                                                                                                         | 285/3996 [23:19<4:31:42,  4.39s/it]
  7%|████████                                                                                                         | 286/3996 [23:23<4:28:26,  4.34s/it]
  7%|████████                                                                                                         | 287/3996 [23:27<4:26:05,  4.30s/it]
  7%|████████▏                                                                                                        | 288/3996 [23:31<4:24:28,  4.28s/it]
  7%|████████▏                                                                                                        | 289/3996 [23:36<4:24:14,  4.28s/it]
  7%|████████▏                                                                                                        | 290/3996 [23:40<4:22:58,  4.26s/it]
  7%|████████▏                                                                                                        | 291/3996 [23:44<4:22:05,  4.24s/it]
  7%|████████▎                                                                                                        | 292/3996 [23:49<4:31:22,  4.40s/it]
  7%|████████▎                                                                                                        | 293/3996 [23:53<4:28:09,  4.34s/it]
  7%|████████▎                                                                                                        | 294/3996 [23:57<4:25:26,  4.30s/it]
  7%|████████▎                                                                                                        | 295/3996 [24:01<4:23:41,  4.28s/it]
  7%|████████▎                                                                                                        | 296/3996 [24:06<4:22:28,  4.26s/it]
  7%|████████▍                                                                                                        | 297/3996 [24:10<4:21:23,  4.24s/it]
  7%|████████▍                                                                                                        | 298/3996 [24:14<4:20:30,  4.23s/it]
  7%|████████▍                                                                                                        | 299/3996 [24:19<4:30:12,  4.39s/it]
  8%|████████▍                                                                                                        | 300/3996 [24:23<4:27:12,  4.34s/it]
                                                                                                                                                           
{'loss': 0.6196, 'grad_norm': 0.18073013424873352, 'learning_rate': 0.00019871528764029667, 'ppl': 1.8582, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4234.51, 'total_tokens': 6668111, 'epoch': 0.23}
+
  8%|████████▍                                                                                                        | 300/3996 [24:23<4:27:12,  4.34s/it]
  8%|████████▌                                                                                                        | 301/3996 [24:27<4:24:51,  4.30s/it]
  8%|████████▌                                                                                                        | 302/3996 [24:31<4:22:51,  4.27s/it]
  8%|████████▌                                                                                                        | 303/3996 [24:36<4:21:40,  4.25s/it]
  8%|████████▌                                                                                                        | 304/3996 [24:40<4:20:59,  4.24s/it]
  8%|████████▌                                                                                                        | 305/3996 [24:44<4:20:23,  4.23s/it]
  8%|████████▋                                                                                                        | 306/3996 [24:49<4:29:50,  4.39s/it]
  8%|████████▋                                                                                                        | 307/3996 [24:53<4:26:25,  4.33s/it]
  8%|████████▋                                                                                                        | 308/3996 [24:57<4:24:10,  4.30s/it]
  8%|████████▋                                                                                                        | 309/3996 [25:01<4:22:30,  4.27s/it]
  8%|████████▊                                                                                                        | 310/3996 [25:06<4:21:02,  4.25s/it]
  8%|████████▊                                                                                                        | 311/3996 [25:10<4:20:09,  4.24s/it]
  8%|████████▊                                                                                                        | 312/3996 [25:14<4:19:28,  4.23s/it]
  8%|████████▊                                                                                                        | 313/3996 [25:19<4:29:15,  4.39s/it]
  8%|████████▉                                                                                                        | 314/3996 [25:23<4:26:08,  4.34s/it]
  8%|████████▉                                                                                                        | 315/3996 [25:27<4:23:42,  4.30s/it]
  8%|████████▉                                                                                                        | 316/3996 [25:31<4:22:00,  4.27s/it]
  8%|████████▉                                                                                                        | 317/3996 [25:36<4:20:56,  4.26s/it]
  8%|████████▉                                                                                                        | 318/3996 [25:40<4:20:10,  4.24s/it]
  8%|█████████                                                                                                        | 319/3996 [25:44<4:19:54,  4.24s/it]
  8%|█████████                                                                                                        | 320/3996 [25:49<4:29:22,  4.40s/it]
  8%|█████████                                                                                                        | 321/3996 [25:53<4:25:53,  4.34s/it]
  8%|█████████                                                                                                        | 322/3996 [25:57<4:23:35,  4.30s/it]
  8%|█████████▏                                                                                                       | 323/3996 [26:02<4:21:38,  4.27s/it]
  8%|█████████▏                                                                                                       | 324/3996 [26:06<4:20:34,  4.26s/it]
  8%|█████████▏                                                                                                       | 325/3996 [26:10<4:19:47,  4.25s/it]
                                                                                                                                                           
{'loss': 0.6181, 'grad_norm': 0.19639697670936584, 'learning_rate': 0.00019837315262594306, 'ppl': 1.8554, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4261.44, 'total_tokens': 7117439, 'epoch': 0.24}
+
  8%|█████████▏                                                                                                       | 325/3996 [26:10<4:19:47,  4.25s/it]
  8%|█████████▏                                                                                                       | 326/3996 [26:14<4:19:18,  4.24s/it]
  8%|█████████▏                                                                                                       | 327/3996 [26:19<4:29:14,  4.40s/it]
  8%|█████████▎                                                                                                       | 328/3996 [26:23<4:25:56,  4.35s/it]
  8%|█████████▎                                                                                                       | 329/3996 [26:27<4:23:19,  4.31s/it]
  8%|█████████▎                                                                                                       | 330/3996 [26:32<4:21:13,  4.28s/it]
  8%|█████████▎                                                                                                       | 331/3996 [26:36<4:20:01,  4.26s/it]
  8%|█████████▍                                                                                                       | 332/3996 [26:40<4:19:11,  4.24s/it]
  8%|█████████▍                                                                                                       | 333/3996 [26:44<4:18:56,  4.24s/it]
  8%|█████████▍                                                                                                       | 334/3996 [26:49<4:29:44,  4.42s/it]
  8%|█████████▍                                                                                                       | 335/3996 [26:53<4:26:02,  4.36s/it]
  8%|█████████▌                                                                                                       | 336/3996 [26:58<4:23:39,  4.32s/it]
  8%|█████████▌                                                                                                       | 337/3996 [27:02<4:21:34,  4.29s/it]
  8%|█████████▌                                                                                                       | 338/3996 [27:06<4:20:03,  4.27s/it]
  8%|█████████▌                                                                                                       | 339/3996 [27:10<4:19:04,  4.25s/it]
  9%|█████████▌                                                                                                       | 340/3996 [27:14<4:18:09,  4.24s/it]
  9%|█████████▋                                                                                                       | 341/3996 [27:19<4:29:17,  4.42s/it]
  9%|█████████▋                                                                                                       | 342/3996 [27:23<4:25:32,  4.36s/it]
  9%|█████████▋                                                                                                       | 343/3996 [27:28<4:22:39,  4.31s/it]
  9%|█████████▋                                                                                                       | 344/3996 [27:32<4:20:51,  4.29s/it]
  9%|█████████▊                                                                                                       | 345/3996 [27:36<4:19:16,  4.26s/it]
  9%|█████████▊                                                                                                       | 346/3996 [27:40<4:18:11,  4.24s/it]
  9%|█████████▊                                                                                                       | 347/3996 [27:45<4:17:35,  4.24s/it]
  9%|█████████▊                                                                                                       | 348/3996 [27:49<4:26:40,  4.39s/it]
  9%|█████████▊                                                                                                       | 349/3996 [27:53<4:23:18,  4.33s/it]
  9%|█████████▉                                                                                                       | 350/3996 [27:58<4:20:53,  4.29s/it]
                                                                                                                                                           
{'loss': 0.6141, 'grad_norm': 0.1670486479997635, 'learning_rate': 0.00019799104120548492, 'ppl': 1.848, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4298.97, 'total_tokens': 7569060, 'epoch': 0.26}
+
  9%|█████████▉                                                                                                       | 350/3996 [27:58<4:20:53,  4.29s/it]
  9%|█████████▉                                                                                                       | 351/3996 [28:02<4:19:04,  4.26s/it]
  9%|█████████▉                                                                                                       | 352/3996 [28:06<4:18:06,  4.25s/it]
  9%|█████████▉                                                                                                       | 353/3996 [28:10<4:17:22,  4.24s/it]
  9%|██████████                                                                                                       | 354/3996 [28:14<4:16:46,  4.23s/it]
  9%|██████████                                                                                                       | 355/3996 [28:19<4:26:16,  4.39s/it]
  9%|██████████                                                                                                       | 356/3996 [28:23<4:23:11,  4.34s/it]
  9%|██████████                                                                                                       | 357/3996 [28:28<4:21:02,  4.30s/it]
  9%|██████████                                                                                                       | 358/3996 [28:32<4:19:20,  4.28s/it]
  9%|██████████▏                                                                                                      | 359/3996 [28:36<4:17:38,  4.25s/it]
  9%|██████████▏                                                                                                      | 360/3996 [28:40<4:16:50,  4.24s/it]
  9%|██████████▏                                                                                                      | 361/3996 [28:45<4:16:11,  4.23s/it]
  9%|██████████▏                                                                                                      | 362/3996 [28:49<4:25:40,  4.39s/it]
  9%|██████████▎                                                                                                      | 363/3996 [28:53<4:22:13,  4.33s/it]
  9%|██████████▎                                                                                                      | 364/3996 [28:58<4:19:57,  4.29s/it]
  9%|██████████▎                                                                                                      | 365/3996 [29:02<4:18:17,  4.27s/it]
  9%|██████████▎                                                                                                      | 366/3996 [29:06<4:17:25,  4.26s/it]
  9%|██████████▍                                                                                                      | 367/3996 [29:10<4:16:40,  4.24s/it]
  9%|██████████▍                                                                                                      | 368/3996 [29:15<4:16:20,  4.24s/it]
  9%|██████████▍                                                                                                      | 369/3996 [29:19<4:25:22,  4.39s/it]
  9%|██████████▍                                                                                                      | 370/3996 [29:24<4:29:19,  4.46s/it]
  9%|██████████▍                                                                                                      | 371/3996 [29:28<4:24:43,  4.38s/it]
  9%|██████████▌                                                                                                      | 372/3996 [29:32<4:21:18,  4.33s/it]
  9%|██████████▌                                                                                                      | 373/3996 [29:37<4:19:16,  4.29s/it]
  9%|██████████▌                                                                                                      | 374/3996 [29:41<4:17:48,  4.27s/it]
  9%|██████████▌                                                                                                      | 375/3996 [29:45<4:16:45,  4.25s/it]
                                                                                                                                                           
{'loss': 0.6075, 'grad_norm': 0.17752495408058167, 'learning_rate': 0.00019756910865951377, 'ppl': 1.8358, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4256.6, 'total_tokens': 8017630, 'epoch': 0.28}
+
  9%|██████████▌                                                                                                      | 375/3996 [29:45<4:16:45,  4.25s/it]
  9%|██████████▋                                                                                                      | 376/3996 [29:50<4:25:49,  4.41s/it]
  9%|██████████▋                                                                                                      | 377/3996 [29:54<4:22:16,  4.35s/it]
  9%|██████████▋                                                                                                      | 378/3996 [29:58<4:19:52,  4.31s/it]
  9%|██████████▋                                                                                                      | 379/3996 [30:02<4:17:58,  4.28s/it]
 10%|██████████▋                                                                                                      | 380/3996 [30:07<4:16:56,  4.26s/it]
 10%|██████████▊                                                                                                      | 381/3996 [30:11<4:16:00,  4.25s/it]
 10%|██████████▊                                                                                                      | 382/3996 [30:15<4:15:26,  4.24s/it]
 10%|██████████▊                                                                                                      | 383/3996 [30:20<4:25:01,  4.40s/it]
 10%|██████████▊                                                                                                      | 384/3996 [30:24<4:21:46,  4.35s/it]
 10%|██████████▉                                                                                                      | 385/3996 [30:28<4:19:23,  4.31s/it]
 10%|██████████▉                                                                                                      | 386/3996 [30:32<4:17:36,  4.28s/it]
 10%|██████████▉                                                                                                      | 387/3996 [30:37<4:16:11,  4.26s/it]
 10%|██████████▉                                                                                                      | 388/3996 [30:41<4:15:22,  4.25s/it]
 10%|███████████                                                                                                      | 389/3996 [30:45<4:14:56,  4.24s/it]
 10%|███████████                                                                                                      | 390/3996 [30:50<4:24:16,  4.40s/it]
 10%|███████████                                                                                                      | 391/3996 [30:54<4:20:51,  4.34s/it]
 10%|███████████                                                                                                      | 392/3996 [30:58<4:18:32,  4.30s/it]
 10%|███████████                                                                                                      | 393/3996 [31:03<4:16:39,  4.27s/it]
 10%|███████████▏                                                                                                     | 394/3996 [31:07<4:15:45,  4.26s/it]
 10%|███████████▏                                                                                                     | 395/3996 [31:11<4:14:47,  4.25s/it]
 10%|███████████▏                                                                                                     | 396/3996 [31:15<4:14:09,  4.24s/it]
 10%|███████████▏                                                                                                     | 397/3996 [31:20<4:23:48,  4.40s/it]
 10%|███████████▎                                                                                                     | 398/3996 [31:24<4:20:33,  4.35s/it]
 10%|███████████▎                                                                                                     | 399/3996 [31:28<4:18:20,  4.31s/it]
 10%|███████████▎                                                                                                     | 400/3996 [31:33<4:16:35,  4.28s/it]
                                                                                                                                                           
{'loss': 0.6108, 'grad_norm': 0.2000180035829544, 'learning_rate': 0.00019710752645093747, 'ppl': 1.8419, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4245.46, 'total_tokens': 8464998, 'epoch': 0.3}
+
 10%|███████████▎                                                                                                     | 400/3996 [31:33<4:16:35,  4.28s/it]
 10%|███████████▎                                                                                                     | 401/3996 [31:37<4:15:08,  4.26s/it]
 10%|███████████▎                                                                                                     | 402/3996 [31:41<4:14:08,  4.24s/it]
 10%|███████████▍                                                                                                     | 403/3996 [31:45<4:13:38,  4.24s/it]
 10%|███████████▍                                                                                                     | 404/3996 [31:50<4:23:06,  4.39s/it]
 10%|███████████▍                                                                                                     | 405/3996 [31:54<4:19:46,  4.34s/it]
 10%|███████████▍                                                                                                     | 406/3996 [31:58<4:17:37,  4.31s/it]
 10%|███████████▌                                                                                                     | 407/3996 [32:03<4:15:45,  4.28s/it]
 10%|███████████▌                                                                                                     | 408/3996 [32:07<4:14:49,  4.26s/it]
 10%|███████████▌                                                                                                     | 409/3996 [32:11<4:13:58,  4.25s/it]
 10%|███████████▌                                                                                                     | 410/3996 [32:15<4:13:25,  4.24s/it]
 10%|███████████▌                                                                                                     | 411/3996 [32:20<4:22:57,  4.40s/it]
 10%|███████████▋                                                                                                     | 412/3996 [32:24<4:19:29,  4.34s/it]
 10%|███████████▋                                                                                                     | 413/3996 [32:29<4:17:10,  4.31s/it]
 10%|███████████▋                                                                                                     | 414/3996 [32:33<4:15:20,  4.28s/it]
 10%|███████████▋                                                                                                     | 415/3996 [32:37<4:14:11,  4.26s/it]
 10%|███████████▊                                                                                                     | 416/3996 [32:41<4:13:32,  4.25s/it]
 10%|███████████▊                                                                                                     | 417/3996 [32:45<4:12:51,  4.24s/it]
 10%|███████████▊                                                                                                     | 418/3996 [32:50<4:22:16,  4.40s/it]
 10%|███████████▊                                                                                                     | 419/3996 [32:54<4:18:56,  4.34s/it]
 11%|███████████▉                                                                                                     | 420/3996 [32:59<4:16:38,  4.31s/it]
 11%|███████████▉                                                                                                     | 421/3996 [33:03<4:15:07,  4.28s/it]
 11%|███████████▉                                                                                                     | 422/3996 [33:07<4:14:10,  4.27s/it]
 11%|███████████▉                                                                                                     | 423/3996 [33:11<4:13:18,  4.25s/it]
 11%|███████████▉                                                                                                     | 424/3996 [33:16<4:13:02,  4.25s/it]
 11%|████████████                                                                                                     | 425/3996 [33:20<4:22:32,  4.41s/it]
                                                                                                                                                           
{'loss': 0.5966, 'grad_norm': 0.17395919561386108, 'learning_rate': 0.00019660648215530206, 'ppl': 1.8159, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3758.92, 'total_tokens': 8914723, 'epoch': 0.32}
+
 11%|████████████                                                                                                     | 425/3996 [33:20<4:22:32,  4.41s/it]
 11%|████████████                                                                                                     | 426/3996 [33:25<4:19:17,  4.36s/it]
 11%|████████████                                                                                                     | 427/3996 [33:29<4:16:47,  4.32s/it]
 11%|████████████                                                                                                     | 428/3996 [33:33<4:14:44,  4.28s/it]
 11%|████████████▏                                                                                                    | 429/3996 [33:37<4:13:37,  4.27s/it]
 11%|████████████▏                                                                                                    | 430/3996 [33:41<4:12:39,  4.25s/it]
 11%|████████████▏                                                                                                    | 431/3996 [33:46<4:11:54,  4.24s/it]
 11%|████████████▏                                                                                                    | 432/3996 [33:50<4:20:59,  4.39s/it]
 11%|████████████▏                                                                                                    | 433/3996 [33:55<4:17:33,  4.34s/it]
 11%|████████████▎                                                                                                    | 434/3996 [33:59<4:15:07,  4.30s/it]
 11%|████████████▎                                                                                                    | 435/3996 [34:03<4:13:38,  4.27s/it]
 11%|████████████▎                                                                                                    | 436/3996 [34:07<4:12:59,  4.26s/it]
 11%|████████████▎                                                                                                    | 437/3996 [34:11<4:12:20,  4.25s/it]
 11%|████████████▍                                                                                                    | 438/3996 [34:16<4:11:26,  4.24s/it]
 11%|████████████▍                                                                                                    | 439/3996 [34:20<4:20:32,  4.39s/it]
 11%|████████████▍                                                                                                    | 440/3996 [34:25<4:17:46,  4.35s/it]
 11%|████████████▍                                                                                                    | 441/3996 [34:29<4:15:07,  4.31s/it]
 11%|████████████▍                                                                                                    | 442/3996 [34:33<4:13:10,  4.27s/it]
 11%|████████████▌                                                                                                    | 443/3996 [34:37<4:12:23,  4.26s/it]
 11%|████████████▌                                                                                                    | 444/3996 [34:42<4:11:56,  4.26s/it]
 11%|████████████▌                                                                                                    | 445/3996 [34:46<4:11:08,  4.24s/it]
 11%|████████████▌                                                                                                    | 446/3996 [34:51<4:20:00,  4.39s/it]
 11%|████████████▋                                                                                                    | 447/3996 [34:55<4:17:02,  4.35s/it]
 11%|████████████▋                                                                                                    | 448/3996 [34:59<4:14:40,  4.31s/it]
 11%|████████████▋                                                                                                    | 449/3996 [35:03<4:12:57,  4.28s/it]
 11%|████████████▋                                                                                                    | 450/3996 [35:07<4:12:08,  4.27s/it]
                                                                                                                                                           
{'loss': 0.6099, 'grad_norm': 0.18785236775875092, 'learning_rate': 0.00019606617938456572, 'ppl': 1.8402, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4200.53, 'total_tokens': 9359638, 'epoch': 0.34}
+
 11%|████████████▋                                                                                                    | 450/3996 [35:07<4:12:08,  4.27s/it]
 11%|████████████▊                                                                                                    | 451/3996 [35:12<4:11:11,  4.25s/it]
 11%|████████████▊                                                                                                    | 452/3996 [35:16<4:10:15,  4.24s/it]
 11%|████████████▊                                                                                                    | 453/3996 [35:21<4:19:24,  4.39s/it]
 11%|████████████▊                                                                                                    | 454/3996 [35:25<4:16:37,  4.35s/it]
 11%|████████████▊                                                                                                    | 455/3996 [35:29<4:14:21,  4.31s/it]
 11%|████████████▉                                                                                                    | 456/3996 [35:33<4:12:15,  4.28s/it]
 11%|████████████▉                                                                                                    | 457/3996 [35:37<4:11:34,  4.27s/it]
 11%|████████████▉                                                                                                    | 458/3996 [35:42<4:10:45,  4.25s/it]
 11%|████████████▉                                                                                                    | 459/3996 [35:46<4:10:04,  4.24s/it]
 12%|█████████████                                                                                                    | 460/3996 [35:51<4:23:00,  4.46s/it]
 12%|█████████████                                                                                                    | 461/3996 [35:55<4:18:38,  4.39s/it]
 12%|█████████████                                                                                                    | 462/3996 [35:59<4:15:29,  4.34s/it]
 12%|█████████████                                                                                                    | 463/3996 [36:04<4:13:02,  4.30s/it]
 12%|█████████████                                                                                                    | 464/3996 [36:08<4:11:43,  4.28s/it]
 12%|█████████████▏                                                                                                   | 465/3996 [36:12<4:10:50,  4.26s/it]
 12%|█████████████▏                                                                                                   | 466/3996 [36:16<4:10:00,  4.25s/it]
 12%|█████████████▏                                                                                                   | 467/3996 [36:21<4:18:59,  4.40s/it]
 12%|█████████████▏                                                                                                   | 468/3996 [36:25<4:15:47,  4.35s/it]
 12%|█████████████▎                                                                                                   | 469/3996 [36:29<4:13:21,  4.31s/it]
 12%|█████████████▎                                                                                                   | 470/3996 [36:34<4:11:28,  4.28s/it]
 12%|█████████████▎                                                                                                   | 471/3996 [36:38<4:10:27,  4.26s/it]
 12%|█████████████▎                                                                                                   | 472/3996 [36:42<4:09:49,  4.25s/it]
 12%|█████████████▍                                                                                                   | 473/3996 [36:46<4:09:21,  4.25s/it]
 12%|█████████████▍                                                                                                   | 474/3996 [36:51<4:18:53,  4.41s/it]
 12%|█████████████▍                                                                                                   | 475/3996 [36:55<4:15:40,  4.36s/it]
                                                                                                                                                           
{'loss': 0.5922, 'grad_norm': 0.17702797055244446, 'learning_rate': 0.0001954868377043559, 'ppl': 1.808, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4265.36, 'total_tokens': 9810837, 'epoch': 0.36}
+
 12%|█████████████▍                                                                                                   | 475/3996 [36:55<4:15:40,  4.36s/it]
 12%|█████████████▍                                                                                                   | 476/3996 [37:00<4:13:09,  4.32s/it]
 12%|█████████████▍                                                                                                   | 477/3996 [37:04<4:11:27,  4.29s/it]
 12%|█████████████▌                                                                                                   | 478/3996 [37:08<4:10:16,  4.27s/it]
 12%|█████████████▌                                                                                                   | 479/3996 [37:12<4:09:23,  4.25s/it]
 12%|█████████████▌                                                                                                   | 480/3996 [37:16<4:08:39,  4.24s/it]
 12%|█████████████▌                                                                                                   | 481/3996 [37:21<4:17:58,  4.40s/it]
 12%|█████████████▋                                                                                                   | 482/3996 [37:25<4:14:54,  4.35s/it]
 12%|█████████████▋                                                                                                   | 483/3996 [37:30<4:12:45,  4.32s/it]
 12%|█████████████▋                                                                                                   | 484/3996 [37:34<4:10:52,  4.29s/it]
 12%|█████████████▋                                                                                                   | 485/3996 [37:38<4:09:41,  4.27s/it]
 12%|█████████████▋                                                                                                   | 486/3996 [37:42<4:08:52,  4.25s/it]
 12%|█████████████▊                                                                                                   | 487/3996 [37:47<4:08:10,  4.24s/it]
 12%|█████████████▊                                                                                                   | 488/3996 [37:51<4:17:39,  4.41s/it]
 12%|█████████████▊                                                                                                   | 489/3996 [37:56<4:14:47,  4.36s/it]
 12%|█████████████▊                                                                                                   | 490/3996 [38:00<4:12:19,  4.32s/it]
 12%|█████████████▉                                                                                                   | 491/3996 [38:04<4:10:23,  4.29s/it]
 12%|█████████████▉                                                                                                   | 492/3996 [38:08<4:09:24,  4.27s/it]
 12%|█████████████▉                                                                                                   | 493/3996 [38:13<4:08:34,  4.26s/it]
 12%|█████████████▉                                                                                                   | 494/3996 [38:17<4:07:32,  4.24s/it]
 12%|█████████████▉                                                                                                   | 495/3996 [38:21<4:16:39,  4.40s/it]
 12%|██████████████                                                                                                   | 496/3996 [38:26<4:18:17,  4.43s/it]
 12%|██████████████                                                                                                   | 497/3996 [38:30<4:14:33,  4.37s/it]
 12%|██████████████                                                                                                   | 498/3996 [38:34<4:11:48,  4.32s/it]
 12%|██████████████                                                                                                   | 499/3996 [38:39<4:09:52,  4.29s/it]
 13%|██████████████▏                                                                                                  | 500/3996 [38:43<4:08:33,  4.27s/it]
                                                                                                                                                           
{'loss': 0.5759, 'grad_norm': 0.19927558302879333, 'learning_rate': 0.00019486869254474337, 'ppl': 1.7787, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4276.25, 'total_tokens': 10261446, 'epoch': 0.38}
+
 13%|██████████████▏                                                                                                  | 500/3996 [38:43<4:08:33,  4.27s/it]
 13%|██████████████▏                                                                                                  | 501/3996 [38:47<4:07:39,  4.25s/it]
 13%|██████████████▏                                                                                                  | 502/3996 [38:52<4:16:25,  4.40s/it]
 13%|██████████████▏                                                                                                  | 503/3996 [38:56<4:13:07,  4.35s/it]
 13%|██████████████▎                                                                                                  | 504/3996 [39:00<4:10:59,  4.31s/it]
 13%|██████████████▎                                                                                                  | 505/3996 [39:04<4:08:55,  4.28s/it]
 13%|██████████████▎                                                                                                  | 506/3996 [39:09<4:08:00,  4.26s/it]
 13%|██████████████▎                                                                                                  | 507/3996 [39:13<4:07:02,  4.25s/it]
 13%|█���████████████▎                                                                                                  | 508/3996 [39:17<4:06:32,  4.24s/it]
 13%|██████████████▍                                                                                                  | 509/3996 [39:22<4:15:31,  4.40s/it]
 13%|██████████████▍                                                                                                  | 510/3996 [39:26<4:12:28,  4.35s/it]
 13%|██████████████▍                                                                                                  | 511/3996 [39:30<4:10:01,  4.30s/it]
 13%|██████████████▍                                                                                                  | 512/3996 [39:35<4:08:21,  4.28s/it]
 13%|██████████████▌                                                                                                  | 513/3996 [39:39<4:07:17,  4.26s/it]
 13%|██████████████▌                                                                                                  | 514/3996 [39:43<4:06:32,  4.25s/it]
 13%|██████████████▌                                                                                                  | 515/3996 [39:47<4:06:01,  4.24s/it]
 13%|██████████████▌                                                                                                  | 516/3996 [39:52<4:14:45,  4.39s/it]
 13%|██████████████▌                                                                                                  | 517/3996 [39:56<4:11:37,  4.34s/it]
 13%|██████████████▋                                                                                                  | 518/3996 [40:00<4:09:19,  4.30s/it]
 13%|██████████████▋                                                                                                  | 519/3996 [40:05<4:07:32,  4.27s/it]
 13%|██████████████▋                                                                                                  | 520/3996 [40:09<4:06:35,  4.26s/it]
 13%|██████████████▋                                                                                                  | 521/3996 [40:13<4:05:33,  4.24s/it]
 13%|██████████████▊                                                                                                  | 522/3996 [40:17<4:04:45,  4.23s/it]
 13%|██████████████▊                                                                                                  | 523/3996 [40:22<4:14:49,  4.40s/it]
 13%|██████████████▊                                                                                                  | 524/3996 [40:26<4:11:11,  4.34s/it]
 13%|██████████████▊                                                                                                  | 525/3996 [40:30<4:08:19,  4.29s/it]
                                                                                                                                                           
{'loss': 0.584, 'grad_norm': 0.1908370852470398, 'learning_rate': 0.0001942119951045692, 'ppl': 1.7932, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4272.28, 'total_tokens': 10707841, 'epoch': 0.39}
+
 13%|██████████████▊                                                                                                  | 525/3996 [40:30<4:08:19,  4.29s/it]
 13%|██████████████▊                                                                                                  | 526/3996 [40:35<4:06:23,  4.26s/it]
 13%|██████████████▉                                                                                                  | 527/3996 [40:39<4:05:20,  4.24s/it]
 13%|██████████████▉                                                                                                  | 528/3996 [40:43<4:04:53,  4.24s/it]
 13%|██████████████▉                                                                                                  | 529/3996 [40:47<4:04:17,  4.23s/it]
 13%|██████████████▉                                                                                                  | 530/3996 [40:52<4:13:25,  4.39s/it]
 13%|███████████████                                                                                                  | 531/3996 [40:56<4:10:23,  4.34s/it]
 13%|███████���███████                                                                                                  | 532/3996 [41:00<4:08:02,  4.30s/it]
 13%|███████████████                                                                                                  | 533/3996 [41:05<4:06:18,  4.27s/it]
 13%|███████████████                                                                                                  | 534/3996 [41:09<4:05:19,  4.25s/it]
 13%|███████████████▏                                                                                                 | 535/3996 [41:13<4:04:32,  4.24s/it]
 13%|███████████████▏                                                                                                 | 536/3996 [41:17<4:05:28,  4.26s/it]
 13%|███████████████▏                                                                                                 | 537/3996 [41:22<4:14:10,  4.41s/it]
 13%|███████████████▏                                                                                                 | 538/3996 [41:26<4:10:54,  4.35s/it]
 13%|███████████████▏                                                                                                 | 539/3996 [41:31<4:08:30,  4.31s/it]
 14%|███████████████▎                                                                                                 | 540/3996 [41:36<4:29:44,  4.68s/it]
 14%|███████████████▎                                                                                                 | 541/3996 [41:40<4:21:39,  4.54s/it]
 14%|███████████████▎                                                                                                 | 542/3996 [41:44<4:15:39,  4.44s/it]
 14%|███████████████▎                                                                                                 | 543/3996 [41:49<4:11:43,  4.37s/it]
 14%|███████████████▍                                                                                                 | 544/3996 [41:53<4:18:20,  4.49s/it]
 14%|███████████████▍                                                                                                 | 545/3996 [41:58<4:13:28,  4.41s/it]
 14%|███████████████▍                                                                                                 | 546/3996 [42:02<4:14:11,  4.42s/it]
 14%|███████████████▍                                                                                                 | 547/3996 [42:06<4:10:48,  4.36s/it]
 14%|███████████████▍                                                                                                 | 548/3996 [42:11<4:08:39,  4.33s/it]
 14%|███████████████▌                                                                                                 | 549/3996 [42:15<4:06:30,  4.29s/it]
 14%|███████████████▌                                                                                                 | 550/3996 [42:19<4:05:05,  4.27s/it]
                                                                                                                                                           
{'loss': 0.5791, 'grad_norm': 0.2064146101474762, 'learning_rate': 0.00019351701224936383, 'ppl': 1.7844, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4250.37, 'total_tokens': 11155384, 'epoch': 0.41}
+
 14%|███████████████▌                                                                                                 | 550/3996 [42:19<4:05:05,  4.27s/it]
 14%|███████████████▌                                                                                                 | 551/3996 [42:24<4:13:53,  4.42s/it]
 14%|███████████████▌                                                                                                 | 552/3996 [42:28<4:10:22,  4.36s/it]
 14%|███████████████▋                                                                                                 | 553/3996 [42:32<4:07:34,  4.31s/it]
 14%|███████████████▋                                                                                                 | 554/3996 [42:36<4:05:48,  4.28s/it]
 14%|███████████████▋                                                                                                 | 555/3996 [42:41<4:04:36,  4.27s/it]
 14%|███████████████▋                                                                                                 | 556/3996 [42:45<4:03:37,  4.25s/it]
 14%|███████████████▊                                                                                                 | 557/3996 [42:49<4:02:36,  4.23s/it]
 14%|███████████████▊                                                                                                 | 558/3996 [42:54<4:11:11,  4.38s/it]
 14%|███████████████▊                                                                                                 | 559/3996 [42:58<4:07:59,  4.33s/it]
 14%|███████████████▊                                                                                                 | 560/3996 [43:02<4:05:36,  4.29s/it]
 14%|███████████████▊                                                                                                 | 561/3996 [43:06<4:03:42,  4.26s/it]
 14%|███████████████▉                                                                                                 | 562/3996 [43:11<4:02:48,  4.24s/it]
 14%|███████████████▉                                                                                                 | 563/3996 [43:15<4:01:52,  4.23s/it]
 14%|███████████████▉                                                                                                 | 564/3996 [43:19<4:01:13,  4.22s/it]
 14%|███████████████▉                                                                                                 | 565/3996 [43:24<4:10:09,  4.37s/it]
 14%|████████████████                                                                                                 | 566/3996 [43:28<4:07:11,  4.32s/it]
 14%|████████████████                                                                                                 | 567/3996 [43:32<4:05:01,  4.29s/it]
 14%|████████████████                                                                                                 | 568/3996 [43:37<4:10:15,  4.38s/it]
 14%|████████████████                                                                                                 | 569/3996 [43:41<4:07:06,  4.33s/it]
 14%|████████████████                                                                                                 | 570/3996 [43:45<4:04:50,  4.29s/it]
 14%|████████████████▏                                                                                                | 571/3996 [43:49<4:03:21,  4.26s/it]
 14%|████████████████▏                                                                                                | 572/3996 [43:54<4:11:25,  4.41s/it]
 14%|████████████████▏                                                                                                | 573/3996 [43:58<4:08:08,  4.35s/it]
 14%|████████████████▏                                                                                                | 574/3996 [44:02<4:05:50,  4.31s/it]
 14%|████████████████▎                                                                                                | 575/3996 [44:07<4:03:46,  4.28s/it]
                                                                                                                                                           
{'loss': 0.5758, 'grad_norm': 0.26748332381248474, 'learning_rate': 0.0001927840264028995, 'ppl': 1.7786, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4256.55, 'total_tokens': 11601192, 'epoch': 0.43}
+
 14%|████████████████▎                                                                                                | 575/3996 [44:07<4:03:46,  4.28s/it]
 14%|████████████████▎                                                                                                | 576/3996 [44:11<4:02:51,  4.26s/it]
 14%|████████████████▎                                                                                                | 577/3996 [44:15<4:01:37,  4.24s/it]
 14%|████████████████▎                                                                                                | 578/3996 [44:19<4:00:59,  4.23s/it]
 14%|████████████████▎                                                                                                | 579/3996 [44:24<4:09:44,  4.39s/it]
 15%|████████████████▍                                                                                                | 580/3996 [44:28<4:06:46,  4.33s/it]
 15%|████████████████▍                                                                                                | 581/3996 [44:32<4:04:45,  4.30s/it]
 15%|████████████████▍                                                                                                | 582/3996 [44:37<4:02:37,  4.26s/it]
 15%|████████████████▍                                                                                                | 583/3996 [44:41<4:01:42,  4.25s/it]
 15%|████████████████▌                                                                                                | 584/3996 [44:45<4:00:59,  4.24s/it]
 15%|████████████████▌                                                                                                | 585/3996 [44:49<4:00:32,  4.23s/it]
 15%|████████████████▌                                                                                                | 586/3996 [44:54<4:08:59,  4.38s/it]
 15%|████████████████▌                                                                                                | 587/3996 [44:58<4:05:54,  4.33s/it]
 15%|████████████████▋                                                                                                | 588/3996 [45:02<4:03:29,  4.29s/it]
 15%|████████████████▋                                                                                                | 589/3996 [45:07<4:01:48,  4.26s/it]
 15%|████████████████▋                                                                                                | 590/3996 [45:11<4:00:56,  4.24s/it]
 15%|████████████████▋                                                                                                | 591/3996 [45:15<3:59:55,  4.23s/it]
 15%|████████████████▋                                                                                                | 592/3996 [45:19<3:59:29,  4.22s/it]
 15%|████████████████▊                                                                                                | 593/3996 [45:24<4:09:23,  4.40s/it]
 15%|████████████████▊                                                                                                | 594/3996 [45:28<4:06:22,  4.35s/it]
 15%|████████████████▊                                                                                                | 595/3996 [45:32<4:03:40,  4.30s/it]
 15%|████████████████▊                                                                                                | 596/3996 [45:37<4:01:42,  4.27s/it]
 15%|████████████████▉                                                                                                | 597/3996 [45:41<4:00:32,  4.25s/it]
 15%|████████████████▉                                                                                                | 598/3996 [45:45<4:00:01,  4.24s/it]
 15%|████████████████▉                                                                                                | 599/3996 [45:49<3:59:27,  4.23s/it]
 15%|████████████████▉                                                                                                | 600/3996 [45:54<4:08:09,  4.38s/it]
                                                                                                                                                           
{'loss': 0.5791, 'grad_norm': 0.17514832317829132, 'learning_rate': 0.00019201333543242036, 'ppl': 1.7844, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3770.83, 'total_tokens': 12048477, 'epoch': 0.45}
+
 15%|████████████████▉                                                                                                | 600/3996 [45:54<4:08:09,  4.38s/it]
 15%|████████████████▉                                                                                                | 601/3996 [45:58<4:05:17,  4.34s/it]
 15%|█████████████████                                                                                                | 602/3996 [46:02<4:03:25,  4.30s/it]
 15%|█████████████████                                                                                                | 603/3996 [46:07<4:01:47,  4.28s/it]
 15%|█████████████████                                                                                                | 604/3996 [46:11<4:00:28,  4.25s/it]
 15%|█████████████████                                                                                                | 605/3996 [46:15<3:59:51,  4.24s/it]
 15%|█████████████████▏                                                                                               | 606/3996 [46:19<4:00:04,  4.25s/it]
 15%|█████████████████▏                                                                                               | 607/3996 [46:24<4:08:43,  4.40s/it]
 15%|█████████████████▏                                                                                               | 608/3996 [46:28<4:05:26,  4.35s/it]
 15%|█████████████████▏                                                                                               | 609/3996 [46:33<4:02:52,  4.30s/it]
 15%|█████████████████▏                                                                                               | 610/3996 [46:37<4:00:50,  4.27s/it]
 15%|█████████████████▎                                                                                               | 611/3996 [46:41<3:59:50,  4.25s/it]
 15%|█████████████████▎                                                                                               | 612/3996 [46:45<3:58:52,  4.24s/it]
 15%|█████████████████▎                                                                                               | 613/3996 [46:49<3:58:22,  4.23s/it]
 15%|█████████████████▎                                                                                               | 614/3996 [46:54<4:07:22,  4.39s/it]
 15%|█████████████████▍                                                                                               | 615/3996 [46:58<4:04:16,  4.33s/it]
 15%|█████████████████▍                                                                                               | 616/3996 [47:03<4:03:15,  4.32s/it]
 15%|█████████████████▍                                                                                               | 617/3996 [47:07<4:01:02,  4.28s/it]
 15%|█████████████████▍                                                                                               | 618/3996 [47:11<3:59:40,  4.26s/it]
 15%|█████████████████▌                                                                                               | 619/3996 [47:15<3:58:45,  4.24s/it]
 16%|█████████████████▌                                                                                               | 620/3996 [47:19<3:58:18,  4.24s/it]
 16%|█████████████████▌                                                                                               | 621/3996 [47:24<4:07:16,  4.40s/it]
 16%|█████████████████▌                                                                                               | 622/3996 [47:28<4:04:00,  4.34s/it]
 16%|█████████████████▌                                                                                               | 623/3996 [47:33<4:01:24,  4.29s/it]
 16%|█████████████████▋                                                                                               | 624/3996 [47:37<3:59:35,  4.26s/it]
 16%|█████████████████▋                                                                                               | 625/3996 [47:41<3:58:36,  4.25s/it]
                                                                                                                                                           
{'loss': 0.5803, 'grad_norm': 0.22069169580936432, 'learning_rate': 0.00019120525252759647, 'ppl': 1.7866, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4179.31, 'total_tokens': 12488141, 'epoch': 0.47}
+
 16%|█████████████████▋                                                                                               | 625/3996 [47:41<3:58:36,  4.25s/it]
 16%|█████████████████▋                                                                                               | 626/3996 [47:45<3:57:46,  4.23s/it]
 16%|█████████████████▋                                                                                               | 627/3996 [47:49<3:57:40,  4.23s/it]
 16%|█████████████████▊                                                                                               | 628/3996 [47:54<4:06:34,  4.39s/it]
 16%|█████████████████▊                                                                                               | 629/3996 [47:58<4:03:30,  4.34s/it]
 16%|█████████████████▊                                                                                               | 630/3996 [48:03<4:01:03,  4.30s/it]
 16%|█████████████████▊                                                                                               | 631/3996 [48:07<3:59:28,  4.27s/it]
 16%|█████████████████▊                                                                                               | 632/3996 [48:11<3:58:44,  4.26s/it]
 16%|█████████████████▉                                                                                               | 633/3996 [48:15<3:58:08,  4.25s/it]
 16%|█████████████████▉                                                                                               | 634/3996 [48:20<3:57:44,  4.24s/it]
 16%|█████████████████▉                                                                                               | 635/3996 [48:24<4:06:20,  4.40s/it]
 16%|█████████████████▉                                                                                               | 636/3996 [48:28<4:03:30,  4.35s/it]
 16%|██████████████████                                                                                               | 637/3996 [48:33<4:01:15,  4.31s/it]
 16%|██████████████████                                                                                               | 638/3996 [48:37<3:59:35,  4.28s/it]
 16%|██████████████████                                                                                               | 639/3996 [48:41<3:59:19,  4.28s/it]
 16%|██████████████████                                                                                               | 640/3996 [48:45<3:58:28,  4.26s/it]
 16%|██████████████████▏                                                                                              | 641/3996 [48:50<3:57:44,  4.25s/it]
 16%|██████████████████▏                                                                                              | 642/3996 [48:54<4:06:33,  4.41s/it]
 16%|██████████████████▏                                                                                              | 643/3996 [48:59<4:03:31,  4.36s/it]
 16%|██████████████████▏                                                                                              | 644/3996 [49:03<4:01:15,  4.32s/it]
 16%|██████████████████▏                                                                                              | 645/3996 [49:07<3:59:27,  4.29s/it]
 16%|██████████████████▎                                                                                              | 646/3996 [49:11<3:58:19,  4.27s/it]
 16%|██████████████████▎                                                                                              | 647/3996 [49:16<3:57:38,  4.26s/it]
 16%|██████████████████▎                                                                                              | 648/3996 [49:20<3:57:02,  4.25s/it]
 16%|██████████████████▎                                                                                              | 649/3996 [49:25<4:05:33,  4.40s/it]
 16%|██████████████████▍                                                                                              | 650/3996 [49:29<4:02:47,  4.35s/it]
                                                                                                                                                           
{'loss': 0.5716, 'grad_norm': 0.20555566251277924, 'learning_rate': 0.00019036010607325138, 'ppl': 1.7711, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4209.96, 'total_tokens': 12934358, 'epoch': 0.49}
+
 16%|██████████████████▍                                                                                              | 650/3996 [49:29<4:02:47,  4.35s/it]
 16%|██████████████████▍                                                                                              | 651/3996 [49:33<4:00:31,  4.31s/it]
 16%|██████████████████▍                                                                                              | 652/3996 [49:37<3:58:50,  4.29s/it]
 16%|██████████████████▍                                                                                              | 653/3996 [49:41<3:57:37,  4.27s/it]
 16%|██████████████████▍                                                                                              | 654/3996 [49:46<3:56:55,  4.25s/it]
 16%|██████████████████▌                                                                                              | 655/3996 [49:50<3:56:22,  4.24s/it]
 16%|██████████████████▌                                                                                              | 656/3996 [49:55<4:04:59,  4.40s/it]
 16%|██████████████████▌                                                                                              | 657/3996 [49:59<4:01:56,  4.35s/it]
 16%|██████████████████▌                                                                                              | 658/3996 [50:03<4:02:00,  4.35s/it]
 16%|██████████████████▋                                                                                              | 659/3996 [50:07<3:59:34,  4.31s/it]
 17%|██████████████████▋                                                                                              | 660/3996 [50:12<3:58:03,  4.28s/it]
 17%|██████████████████▋                                                                                              | 661/3996 [50:16<3:56:52,  4.26s/it]
 17%|██████████████████▋                                                                                              | 662/3996 [50:20<3:56:02,  4.25s/it]
 17%|██████████████████▋                                                                                              | 663/3996 [50:25<4:04:30,  4.40s/it]
 17%|██████████████████▊                                                                                              | 664/3996 [50:29<4:01:27,  4.35s/it]
 17%|██████████████████▊                                                                                              | 665/3996 [50:33<3:58:53,  4.30s/it]
 17%|██████████████████▊                                                                                              | 666/3996 [50:37<3:56:47,  4.27s/it]
 17%|██████████████████▊                                                                                              | 667/3996 [50:42<3:55:45,  4.25s/it]
 17%|██████████████████▉                                                                                              | 668/3996 [50:46<3:55:01,  4.24s/it]
 17%|██████████████████▉                                                                                              | 669/3996 [50:50<3:54:46,  4.23s/it]
 17%|██████████████████▉                                                                                              | 670/3996 [50:55<4:03:17,  4.39s/it]
 17%|██████████████████▉                                                                                              | 671/3996 [50:59<3:59:49,  4.33s/it]
 17%|███████████████████                                                                                              | 672/3996 [51:03<3:57:46,  4.29s/it]
 17%|███████████████████                                                                                              | 673/3996 [51:07<3:56:06,  4.26s/it]
 17%|███████████████████                                                                                              | 674/3996 [51:12<3:55:10,  4.25s/it]
 17%|███████████████████                                                                                              | 675/3996 [51:16<3:54:28,  4.24s/it]
                                                                                                                                                           
{'loss': 0.5608, 'grad_norm': 0.19018156826496124, 'learning_rate': 0.00018947823951591478, 'ppl': 1.7521, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4226.4, 'total_tokens': 13378983, 'epoch': 0.51}
+
 17%|███████████████████                                                                                              | 675/3996 [51:16<3:54:28,  4.24s/it]
 17%|███████████████████                                                                                              | 676/3996 [51:20<3:54:05,  4.23s/it]
 17%|███████████████████▏                                                                                             | 677/3996 [51:25<4:04:31,  4.42s/it]
 17%|███████████████████▏                                                                                             | 678/3996 [51:29<4:01:10,  4.36s/it]
 17%|███████████████████▏                                                                                             | 679/3996 [51:33<3:58:47,  4.32s/it]
 17%|███████████████████▏                                                                                             | 680/3996 [51:38<3:56:53,  4.29s/it]
 17%|███████████████████▎                                                                                             | 681/3996 [51:42<3:55:46,  4.27s/it]
 17%|███████████████████▎                                                                                             | 682/3996 [51:46<3:54:52,  4.25s/it]
 17%|███████████████████▎                                                                                             | 683/3996 [51:50<3:54:41,  4.25s/it]
 17%|███████████████████▎                                                                                             | 684/3996 [51:55<4:02:53,  4.40s/it]
 17%|███████████████████▎                                                                                             | 685/3996 [51:59<3:59:35,  4.34s/it]
 17%|███████████████████▍                                                                                             | 686/3996 [52:03<3:57:36,  4.31s/it]
 17%|███████████████████▍                                                                                             | 687/3996 [52:08<3:56:01,  4.28s/it]
 17%|███████████████████▍                                                                                             | 688/3996 [52:12<3:55:07,  4.26s/it]
 17%|███████████████████▍                                                                                             | 689/3996 [52:16<3:54:26,  4.25s/it]
 17%|███████████████████▌                                                                                             | 690/3996 [52:20<3:53:41,  4.24s/it]
 17%|███████████████████▌                                                                                             | 691/3996 [52:25<4:02:20,  4.40s/it]
 17%|███████████████████▌                                                                                             | 692/3996 [52:29<3:59:38,  4.35s/it]
 17%|███████████████████▌                                                                                             | 693/3996 [52:34<3:57:26,  4.31s/it]
 17%|███████████████████▋                                                                                             | 694/3996 [52:38<3:56:26,  4.30s/it]
 17%|███████████████████▋                                                                                             | 695/3996 [52:42<3:55:26,  4.28s/it]
 17%|███████████████████▋                                                                                             | 696/3996 [52:46<3:54:43,  4.27s/it]
 17%|███████████████████▋                                                                                             | 697/3996 [52:51<3:54:16,  4.26s/it]
 17%|███████████████████▋                                                                                             | 698/3996 [52:55<4:02:35,  4.41s/it]
 17%|███████████████████▊                                                                                             | 699/3996 [53:00<3:59:42,  4.36s/it]
 18%|███████████████████▊                                                                                             | 700/3996 [53:04<3:57:23,  4.32s/it]
                                                                                                                                                           
{'loss': 0.5667, 'grad_norm': 0.17173859477043152, 'learning_rate': 0.00018856001122425416, 'ppl': 1.7624, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4265.57, 'total_tokens': 13829519, 'epoch': 0.53}
+
 18%|███████████████████▊                                                                                             | 700/3996 [53:04<3:57:23,  4.32s/it]
 18%|███████████████████▊                                                                                             | 701/3996 [53:08<3:55:37,  4.29s/it]
 18%|███████████████████▊                                                                                             | 702/3996 [53:12<3:54:55,  4.28s/it]
 18%|███████████████████▉                                                                                             | 703/3996 [53:17<3:54:02,  4.26s/it]
 18%|███████████████████▉                                                                                             | 704/3996 [53:21<3:53:24,  4.25s/it]
 18%|███████████████████▉                                                                                             | 705/3996 [53:26<4:01:46,  4.41s/it]
 18%|███████████████████▉                                                                                             | 706/3996 [53:30<3:58:52,  4.36s/it]
 18%|███████████████████▉                                                                                             | 707/3996 [53:34<3:56:16,  4.31s/it]
 18%|████████████████████                                                                                             | 708/3996 [53:38<3:54:44,  4.28s/it]
 18%|████████████████████                                                                                             | 709/3996 [53:42<3:53:34,  4.26s/it]
 18%|████████████████████                                                                                             | 710/3996 [53:47<3:52:49,  4.25s/it]
 18%|████████████████████                                                                                             | 711/3996 [53:51<3:52:28,  4.25s/it]
 18%|████████████████████▏                                                                                            | 712/3996 [53:56<4:00:53,  4.40s/it]
 18%|████████████████████▏                                                                                            | 713/3996 [54:00<3:57:51,  4.35s/it]
 18%|████████████████████▏                                                                                            | 714/3996 [54:04<3:55:46,  4.31s/it]
 18%|████████████████████▏                                                                                            | 715/3996 [54:08<3:54:13,  4.28s/it]
 18%|████████████████████▏                                                                                            | 716/3996 [54:13<3:53:25,  4.27s/it]
 18%|████████████████████▎                                                                                            | 717/3996 [54:17<3:52:29,  4.25s/it]
 18%|████████████████████▎                                                                                            | 718/3996 [54:21<3:51:54,  4.24s/it]
 18%|████████████████████▎                                                                                            | 719/3996 [54:26<4:00:52,  4.41s/it]
 18%|████████████████████▎                                                                                            | 720/3996 [54:30<3:58:01,  4.36s/it]
 18%|████████████████████▍                                                                                            | 721/3996 [54:34<3:55:40,  4.32s/it]
 18%|████████████████████▍                                                                                            | 722/3996 [54:38<3:53:52,  4.29s/it]
 18%|████████████████████▍                                                                                            | 723/3996 [54:43<3:52:45,  4.27s/it]
 18%|████████████████████▍                                                                                            | 724/3996 [54:47<3:51:59,  4.25s/it]
 18%|████████████████████▌                                                                                            | 725/3996 [54:51<3:51:28,  4.25s/it]
                                                                                                                                                           
{'loss': 0.565, 'grad_norm': 0.17706550657749176, 'learning_rate': 0.0001876057943434428, 'ppl': 1.7594, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4281.61, 'total_tokens': 14281879, 'epoch': 0.54}
+
 18%|████████████████████▌                                                                                            | 725/3996 [54:51<3:51:28,  4.25s/it]
 18%|████████████████████▌                                                                                            | 726/3996 [54:56<3:59:55,  4.40s/it]
 18%|████████████████████▌                                                                                            | 727/3996 [55:00<3:56:38,  4.34s/it]
 18%|████████████████████▌                                                                                            | 728/3996 [55:04<3:54:39,  4.31s/it]
 18%|████████████████████▌                                                                                            | 729/3996 [55:09<3:53:07,  4.28s/it]
 18%|████████████████████▋                                                                                            | 730/3996 [55:13<3:52:17,  4.27s/it]
 18%|████████████████████▋                                                                                            | 731/3996 [55:17<3:51:22,  4.25s/it]
 18%|████████████████████▋                                                                                            | 732/3996 [55:21<3:50:37,  4.24s/it]
 18%|████████████████████▋                                                                                            | 733/3996 [55:26<3:59:10,  4.40s/it]
 18%|████████████████████▊                                                                                            | 734/3996 [55:30<3:56:34,  4.35s/it]
 18%|████████████████████▊                                                                                            | 735/3996 [55:34<3:54:23,  4.31s/it]
 18%|████████████████████▊                                                                                            | 736/3996 [55:39<3:52:28,  4.28s/it]
 18%|████████████████████▊                                                                                            | 737/3996 [55:43<3:51:26,  4.26s/it]
 18%|████████████████████▊                                                                                            | 738/3996 [55:47<3:50:33,  4.25s/it]
 18%|████████████████████▉                                                                                            | 739/3996 [55:51<3:50:03,  4.24s/it]
 19%|████████████████████▉                                                                                            | 740/3996 [55:56<3:58:28,  4.39s/it]
 19%|████████████████████▉                                                                                            | 741/3996 [56:00<3:55:26,  4.34s/it]
 19%|████████████████████▉                                                                                            | 742/3996 [56:04<3:53:27,  4.30s/it]
 19%|█████████████████████                                                                                            | 743/3996 [56:09<3:51:56,  4.28s/it]
 19%|████████████████���████                                                                                            | 744/3996 [56:13<3:50:59,  4.26s/it]
 19%|█████████████████████                                                                                            | 745/3996 [56:17<3:49:56,  4.24s/it]
 19%|█████████████████████                                                                                            | 746/3996 [56:21<3:49:14,  4.23s/it]
 19%|█████████████████████                                                                                            | 747/3996 [56:26<3:57:52,  4.39s/it]
 19%|█████████████████████▏                                                                                           | 748/3996 [56:30<3:54:56,  4.34s/it]
 19%|█████████████████████▏                                                                                           | 749/3996 [56:34<3:52:42,  4.30s/it]
 19%|█████████████████████▏                                                                                           | 750/3996 [56:39<3:51:01,  4.27s/it]
                                                                                                                                                           
{'loss': 0.5666, 'grad_norm': 0.18528586626052856, 'learning_rate': 0.00018661597664352284, 'ppl': 1.7623, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4229.32, 'total_tokens': 14725919, 'epoch': 0.56}
+
 19%|█████████████████████▏                                                                                           | 750/3996 [56:39<3:51:01,  4.27s/it]
 19%|█████████████████████▏                                                                                           | 751/3996 [56:43<3:50:03,  4.25s/it]
 19%|█████████████████████▎                                                                                           | 752/3996 [56:47<3:49:03,  4.24s/it]
 19%|█████████████████████▎                                                                                           | 753/3996 [56:51<3:48:48,  4.23s/it]
 19%|█████████████████████▎                                                                                           | 754/3996 [56:56<3:57:12,  4.39s/it]
 19%|█████████████████████▎                                                                                           | 755/3996 [57:00<3:54:12,  4.34s/it]
 19%|█████████████████████▍                                                                                           | 756/3996 [57:04<3:51:54,  4.29s/it]
 19%|█████████████████████▍                                                                                           | 757/3996 [57:09<3:50:24,  4.27s/it]
 19%|█████████████████████▍                                                                                           | 758/3996 [57:13<3:49:31,  4.25s/it]
 19%|█████████████████████▍                                                                                           | 759/3996 [57:17<3:48:45,  4.24s/it]
 19%|█████████████████████▍                                                                                           | 760/3996 [57:21<3:48:18,  4.23s/it]
 19%|█████████████████████▌                                                                                           | 761/3996 [57:26<3:56:36,  4.39s/it]
 19%|█████████████████████▌                                                                                           | 762/3996 [57:30<3:53:55,  4.34s/it]
 19%|█████████████████████▌                                                                                           | 763/3996 [57:35<3:51:37,  4.30s/it]
 19%|█████████████████████▌                                                                                           | 764/3996 [57:39<3:50:06,  4.27s/it]
 19%|█████████████████████▋                                                                                           | 765/3996 [57:43<3:48:53,  4.25s/it]
 19%|█████████████████████▋                                                                                           | 766/3996 [57:47<3:48:07,  4.24s/it]
 19%|█████████████████████▋                                                                                           | 767/3996 [57:51<3:47:41,  4.23s/it]
 19%|█████████████████████▋                                                                                           | 768/3996 [57:56<3:56:01,  4.39s/it]
 19%|█████████████████████▋                                                                                           | 769/3996 [58:00<3:52:56,  4.33s/it]
 19%|█████████████████████▊                                                                                           | 770/3996 [58:05<3:50:50,  4.29s/it]
 19%|█████████████████████▊                                                                                           | 771/3996 [58:09<3:49:21,  4.27s/it]
 19%|█████████████████████▊                                                                                           | 772/3996 [58:13<3:48:27,  4.25s/it]
 19%|█████████████████████▊                                                                                           | 773/3996 [58:17<3:47:49,  4.24s/it]
 19%|█████████████████████▉                                                                                           | 774/3996 [58:21<3:47:25,  4.23s/it]
 19%|█████████████████████▉                                                                                           | 775/3996 [58:26<3:55:48,  4.39s/it]
                                                                                                                                                           
{'loss': 0.5633, 'grad_norm': 0.16790929436683655, 'learning_rate': 0.00018559096036182516, 'ppl': 1.7565, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3775.0, 'total_tokens': 15175146, 'epoch': 0.58}
+
 19%|█████████████████████▉                                                                                           | 775/3996 [58:26<3:55:48,  4.39s/it]
 19%|█████████████████████▉                                                                                           | 776/3996 [58:30<3:53:02,  4.34s/it]
 19%|█████████████████████▉                                                                                           | 777/3996 [58:35<3:50:49,  4.30s/it]
 19%|██████████████████████                                                                                           | 778/3996 [58:39<3:49:08,  4.27s/it]
 19%|██████████████████████                                                                                           | 779/3996 [58:43<3:47:58,  4.25s/it]
 20%|██████████████████████                                                                                           | 780/3996 [58:47<3:48:19,  4.26s/it]
 20%|██████████████████████                                                                                           | 781/3996 [58:51<3:47:36,  4.25s/it]
 20%|██████████████████████                                                                                           | 782/3996 [58:56<3:55:25,  4.39s/it]
 20%|██████████████████████▏                                                                                          | 783/3996 [59:00<3:52:25,  4.34s/it]
 20%|██████████████████████▏                                                                                          | 784/3996 [59:05<3:50:20,  4.30s/it]
 20%|██████████████████████▏                                                                                          | 785/3996 [59:09<3:48:59,  4.28s/it]
 20%|██████████████████████▏                                                                                          | 786/3996 [59:13<3:48:12,  4.27s/it]
 20%|██████████████████████▎                                                                                          | 787/3996 [59:17<3:47:08,  4.25s/it]
 20%|██████████████████████▎                                                                                          | 788/3996 [59:22<3:46:35,  4.24s/it]
 20%|██████████████████████▎                                                                                          | 789/3996 [59:26<3:54:49,  4.39s/it]
 20%|██████████████████████▎                                                                                          | 790/3996 [59:31<3:52:06,  4.34s/it]
 20%|██████████████████████▎                                                                                          | 791/3996 [59:35<3:50:11,  4.31s/it]
 20%|██████████████████████▍                                                                                          | 792/3996 [59:39<3:48:34,  4.28s/it]
 20%|██████████████████████▍                                                                                          | 793/3996 [59:43<3:47:20,  4.26s/it]
 20%|██████████████████████▍                                                                                          | 794/3996 [59:47<3:46:34,  4.25s/it]
 20%|██████████████████████▍                                                                                          | 795/3996 [59:52<3:46:10,  4.24s/it]
 20%|██████████████████████▌                                                                                          | 796/3996 [59:56<3:54:22,  4.39s/it]
 20%|██████████████████████▏                                                                                        | 797/3996 [1:00:01<3:51:21,  4.34s/it]
 20%|██████████████████████▏                                                                                        | 798/3996 [1:00:05<3:49:22,  4.30s/it]
 20%|██████████████████████▏                                                                                        | 799/3996 [1:00:09<3:47:55,  4.28s/it]
 20%|██████████████████████▏                                                                                        | 800/3996 [1:00:13<3:46:54,  4.26s/it]
                                                                                                                                                           
{'loss': 0.5664, 'grad_norm': 0.17511805891990662, 'learning_rate': 0.00018453116203951005, 'ppl': 1.7619, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4218.07, 'total_tokens': 15619901, 'epoch': 0.6}
+
 20%|██████████████████████▏                                                                                        | 800/3996 [1:00:13<3:46:54,  4.26s/it]
 20%|██████████████████████▎                                                                                        | 801/3996 [1:00:17<3:46:00,  4.24s/it]
 20%|██████████████████████▎                                                                                        | 802/3996 [1:00:22<3:45:21,  4.23s/it]
 20%|██████████████████████▎                                                                                        | 803/3996 [1:00:26<3:53:37,  4.39s/it]
 20%|██████████████████████▎                                                                                        | 804/3996 [1:00:31<3:51:07,  4.34s/it]
 20%|██████████████████████▎                                                                                        | 805/3996 [1:00:35<3:49:09,  4.31s/it]
 20%|██████████████████████▍                                                                                        | 806/3996 [1:00:39<3:47:38,  4.28s/it]
 20%|██████████████████████▍                                                                                        | 807/3996 [1:00:43<3:46:32,  4.26s/it]
 20%|██████████████████████▍                                                                                        | 808/3996 [1:00:48<3:45:56,  4.25s/it]
 20%|██████████████████████▍                                                                                        | 809/3996 [1:00:52<3:45:14,  4.24s/it]
 20%|██████████████████████▌                                                                                        | 810/3996 [1:00:56<3:53:25,  4.40s/it]
 20%|██████████████████████▌                                                                                        | 811/3996 [1:01:01<3:50:06,  4.33s/it]
 20%|██████████████████████▌                                                                                        | 812/3996 [1:01:05<3:48:11,  4.30s/it]
 20%|██████████████████████▌                                                                                        | 813/3996 [1:01:09<3:46:46,  4.27s/it]
 20%|██████████████████████▌                                                                                        | 814/3996 [1:01:13<3:45:57,  4.26s/it]
 20%|██████████████████████▋                                                                                        | 815/3996 [1:01:18<3:45:15,  4.25s/it]
 20%|██████████████████████▋                                                                                        | 816/3996 [1:01:22<3:44:37,  4.24s/it]
 20%|██████████████████████▋                                                                                        | 817/3996 [1:01:27<3:52:45,  4.39s/it]
 20%|██████████████████████▋                                                                                        | 818/3996 [1:01:31<3:49:55,  4.34s/it]
 20%|██████████████████████▊                                                                                        | 819/3996 [1:01:35<3:47:46,  4.30s/it]
 21%|██████████████████████▊                                                                                        | 820/3996 [1:01:39<3:46:14,  4.27s/it]
 21%|██████████████████████▊                                                                                        | 821/3996 [1:01:43<3:45:05,  4.25s/it]
 21%|██████████████████████▊                                                                                        | 822/3996 [1:01:48<3:44:27,  4.24s/it]
 21%|██████████████████████▊                                                                                        | 823/3996 [1:01:52<3:44:00,  4.24s/it]
 21%|██████████████████████▉                                                                                        | 824/3996 [1:01:57<3:52:08,  4.39s/it]
 21%|██████████████████████▉                                                                                        | 825/3996 [1:02:01<3:49:20,  4.34s/it]
                                                                                                                                                           
{'loss': 0.5646, 'grad_norm': 0.19853387773036957, 'learning_rate': 0.0001834370123522954, 'ppl': 1.7587, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4230.84, 'total_tokens': 16066102, 'epoch': 0.62}
+
 21%|██████████████████████▉                                                                                        | 825/3996 [1:02:01<3:49:20,  4.34s/it]
 21%|██████████████████████▉                                                                                        | 826/3996 [1:02:05<3:47:07,  4.30s/it]
 21%|██████████████████████▉                                                                                        | 827/3996 [1:02:09<3:45:30,  4.27s/it]
 21%|███████████████████████                                                                                        | 828/3996 [1:02:13<3:44:26,  4.25s/it]
 21%|███████████████████████                                                                                        | 829/3996 [1:02:18<3:43:43,  4.24s/it]
 21%|███████████████████████                                                                                        | 830/3996 [1:02:22<3:43:04,  4.23s/it]
 21%|███████████████████████                                                                                        | 831/3996 [1:02:27<3:51:08,  4.38s/it]
 21%|███████████████████████                                                                                        | 832/3996 [1:02:31<3:48:39,  4.34s/it]
 21%|█████���█████████████████▏                                                                                       | 833/3996 [1:02:35<3:46:31,  4.30s/it]
 21%|███████████████████████▏                                                                                       | 834/3996 [1:02:39<3:45:08,  4.27s/it]
 21%|███████████████████████▏                                                                                       | 835/3996 [1:02:43<3:44:06,  4.25s/it]
 21%|███████████████████████▏                                                                                       | 836/3996 [1:02:48<3:43:34,  4.25s/it]
 21%|███████████████████████▎                                                                                       | 837/3996 [1:02:52<3:42:50,  4.23s/it]
 21%|███████████████████████▎                                                                                       | 838/3996 [1:02:57<3:52:24,  4.42s/it]
 21%|███████████████████████▎                                                                                       | 839/3996 [1:03:01<3:49:08,  4.35s/it]
 21%|███████████████████████▎                                                                                       | 840/3996 [1:03:05<3:46:50,  4.31s/it]
 21%|███████████████████████▎                                                                                       | 841/3996 [1:03:09<3:45:08,  4.28s/it]
 21%|███████████████████████▍                                                                                       | 842/3996 [1:03:14<3:44:08,  4.26s/it]
 21%|███████████████████████▍                                                                                       | 843/3996 [1:03:18<3:43:14,  4.25s/it]
 21%|███████████████████████▍                                                                                       | 844/3996 [1:03:22<3:42:25,  4.23s/it]
 21%|███████████████████████▍                                                                                       | 845/3996 [1:03:27<3:50:14,  4.38s/it]
 21%|███████████████████████▌                                                                                       | 846/3996 [1:03:31<3:47:36,  4.34s/it]
 21%|███████████████████████▌                                                                                       | 847/3996 [1:03:35<3:45:31,  4.30s/it]
 21%|███████████████████████▌                                                                                       | 848/3996 [1:03:39<3:44:06,  4.27s/it]
 21%|███████████████████████▌                                                                                       | 849/3996 [1:03:44<3:43:06,  4.25s/it]
 21%|███████████████████████▌                                                                                       | 850/3996 [1:03:48<3:42:23,  4.24s/it]
                                                                                                                                                           
{'loss': 0.552, 'grad_norm': 0.18872258067131042, 'learning_rate': 0.00018230895593544056, 'ppl': 1.7367, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4222.33, 'total_tokens': 16510696, 'epoch': 0.64}
+
 21%|███████████████████████▌                                                                                       | 850/3996 [1:03:48<3:42:23,  4.24s/it]
 21%|███████████████████████▋                                                                                       | 851/3996 [1:03:52<3:42:04,  4.24s/it]
 21%|███████████████████████▋                                                                                       | 852/3996 [1:03:57<3:50:08,  4.39s/it]
 21%|███████████████████████▋                                                                                       | 853/3996 [1:04:01<3:47:16,  4.34s/it]
 21%|███████████████████████▋                                                                                       | 854/3996 [1:04:05<3:45:08,  4.30s/it]
 21%|███████████████████████▊                                                                                       | 855/3996 [1:04:09<3:43:31,  4.27s/it]
 21%|███████████████████████▊                                                                                       | 856/3996 [1:04:14<3:42:37,  4.25s/it]
 21%|███████████████████████▊                                                                                       | 857/3996 [1:04:18<3:41:53,  4.24s/it]
 21%|███████████████████████▊                                                                                       | 858/3996 [1:04:22<3:41:19,  4.23s/it]
 21%|███████████████████████▊                                                                                       | 859/3996 [1:04:27<3:49:29,  4.39s/it]
 22%|███████████████████████▉                                                                                       | 860/3996 [1:04:31<3:46:38,  4.34s/it]
 22%|███████████████████████▉                                                                                       | 861/3996 [1:04:35<3:44:39,  4.30s/it]
 22%|███████████████████████▉                                                                                       | 862/3996 [1:04:39<3:43:14,  4.27s/it]
 22%|███████████████████████▉                                                                                       | 863/3996 [1:04:44<3:42:18,  4.26s/it]
 22%|████████████████████████                                                                                       | 864/3996 [1:04:48<3:41:24,  4.24s/it]
 22%|████████████████████████                                                                                       | 865/3996 [1:04:52<3:41:06,  4.24s/it]
 22%|████████████████████████                                                                                       | 866/3996 [1:04:57<3:48:57,  4.39s/it]
 22%|████████████████████████                                                                                       | 867/3996 [1:05:01<3:46:20,  4.34s/it]
 22%|████████████████████████                                                                                       | 868/3996 [1:05:05<3:44:29,  4.31s/it]
 22%|████████████████████████▏                                                                                      | 869/3996 [1:05:09<3:42:54,  4.28s/it]
 22%|████████████████████████▏                                                                                      | 870/3996 [1:05:14<3:41:57,  4.26s/it]
 22%|████████████████████████▏                                                                                      | 871/3996 [1:05:18<3:41:20,  4.25s/it]
 22%|████████████████████████▏                                                                                      | 872/3996 [1:05:22<3:40:41,  4.24s/it]
 22%|████████████████████████▎                                                                                      | 873/3996 [1:05:27<3:48:40,  4.39s/it]
 22%|████████████████████████▎                                                                                      | 874/3996 [1:05:31<3:51:20,  4.45s/it]
 22%|████████████████████████▎                                                                                      | 875/3996 [1:05:36<3:47:46,  4.38s/it]
                                                                                                                                                           
{'loss': 0.5607, 'grad_norm': 0.9702818989753723, 'learning_rate': 0.0001811474512030578, 'ppl': 1.7519, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4200.39, 'total_tokens': 16953918, 'epoch': 0.66}
+
 22%|████████████████████████▎                                                                                      | 875/3996 [1:05:36<3:47:46,  4.38s/it]
 22%|████████████████████████▎                                                                                      | 876/3996 [1:05:40<3:44:55,  4.33s/it]
 22%|████████████████████████▎                                                                                      | 877/3996 [1:05:44<3:43:01,  4.29s/it]
 22%|████████████████████████▍                                                                                      | 878/3996 [1:05:48<3:41:49,  4.27s/it]
 22%|████████████████████████▍                                                                                      | 879/3996 [1:05:53<3:40:59,  4.25s/it]
 22%|████████████████████████▍                                                                                      | 880/3996 [1:05:57<3:48:52,  4.41s/it]
 22%|████████████████████████▍                                                                                      | 881/3996 [1:06:01<3:45:45,  4.35s/it]
 22%|████████████████████████▌                                                                                      | 882/3996 [1:06:06<3:43:21,  4.30s/it]
 22%|████████████████████████▌                                                                                      | 883/3996 [1:06:10<3:41:56,  4.28s/it]
 22%|████████████████████████▌                                                                                      | 884/3996 [1:06:14<3:40:59,  4.26s/it]
 22%|████████████████████████▌                                                                                      | 885/3996 [1:06:18<3:40:20,  4.25s/it]
 22%|████████████████████████▌                                                                                      | 886/3996 [1:06:23<3:39:34,  4.24s/it]
 22%|████████████████████████▋                                                                                      | 887/3996 [1:06:27<3:47:27,  4.39s/it]
 22%|████████████████████████▋                                                                                      | 888/3996 [1:06:32<3:45:49,  4.36s/it]
 22%|████████████████████████▋                                                                                      | 889/3996 [1:06:36<3:43:38,  4.32s/it]
 22%|████████████████████████▋                                                                                      | 890/3996 [1:06:40<3:41:54,  4.29s/it]
 22%|████████████████████████▊                                                                                      | 891/3996 [1:06:44<3:40:50,  4.27s/it]
 22%|████████████████████████▊                                                                                      | 892/3996 [1:06:48<3:40:09,  4.26s/it]
 22%|████████████████████████▊                                                                                      | 893/3996 [1:06:53<3:39:30,  4.24s/it]
 22%|████████████████████████▊                                                                                      | 894/3996 [1:06:57<3:47:24,  4.40s/it]
 22%|████████████████████████▊                                                                                      | 895/3996 [1:07:02<3:44:26,  4.34s/it]
 22%|████████████████████████▉                                                                                      | 896/3996 [1:07:06<3:42:19,  4.30s/it]
 22%|████████████████████████▉                                                                                      | 897/3996 [1:07:10<3:41:39,  4.29s/it]
 22%|████████████████████████▉                                                                                      | 898/3996 [1:07:14<3:40:32,  4.27s/it]
 22%|████████████████████████▉                                                                                      | 899/3996 [1:07:19<3:39:42,  4.26s/it]
 23%|█████████████████████████                                                                                      | 900/3996 [1:07:23<3:38:50,  4.24s/it]
                                                                                                                                                           
{'loss': 0.564, 'grad_norm': 0.17479568719863892, 'learning_rate': 0.00017995297016182405, 'ppl': 1.7577, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4210.15, 'total_tokens': 17396453, 'epoch': 0.68}
+
 23%|█████████████████████████                                                                                      | 900/3996 [1:07:23<3:38:50,  4.24s/it]
 23%|█████████████████████████                                                                                      | 901/3996 [1:07:28<3:46:46,  4.40s/it]
 23%|█████████████████████████                                                                                      | 902/3996 [1:07:32<3:44:07,  4.35s/it]
 23%|█████████████████████████                                                                                      | 903/3996 [1:07:36<3:41:51,  4.30s/it]
 23%|█████████████████████████                                                                                      | 904/3996 [1:07:40<3:40:19,  4.28s/it]
 23%|█████████████████████████▏                                                                                     | 905/3996 [1:07:44<3:39:16,  4.26s/it]
 23%|█████████████████████████▏                                                                                     | 906/3996 [1:07:49<3:38:40,  4.25s/it]
 23%|█████████████████████████▏                                                                                     | 907/3996 [1:07:53<3:38:09,  4.24s/it]
 23%|█████████████████████████▏                                                                                     | 908/3996 [1:07:58<3:45:52,  4.39s/it]
 23%|█████████████████████████▎                                                                                     | 909/3996 [1:08:02<3:43:11,  4.34s/it]
 23%|█████████████████████████▎                                                                                     | 910/3996 [1:08:06<3:41:04,  4.30s/it]
 23%|█████████████████████████▎                                                                                     | 911/3996 [1:08:10<3:39:31,  4.27s/it]
 23%|█████████████████████████▎                                                                                     | 912/3996 [1:08:14<3:38:31,  4.25s/it]
 23%|█████████████████████████▎                                                                                     | 913/3996 [1:08:19<3:38:07,  4.25s/it]
 23%|█████████████████████████▍                                                                                     | 914/3996 [1:08:23<3:37:36,  4.24s/it]
 23%|█████████████████████████▍                                                                                     | 915/3996 [1:08:28<3:45:29,  4.39s/it]
 23%|█████████████████████████▍                                                                                     | 916/3996 [1:08:32<3:42:48,  4.34s/it]
 23%|█████████████████████████▍                                                                                     | 917/3996 [1:08:36<3:40:52,  4.30s/it]
 23%|█████████████████████████▌                                                                                     | 918/3996 [1:08:40<3:39:19,  4.28s/it]
 23%|█████████████████████████▌                                                                                     | 919/3996 [1:08:44<3:38:17,  4.26s/it]
 23%|█████████████████████████▌                                                                                     | 920/3996 [1:08:49<3:37:30,  4.24s/it]
 23%|█████████████████████████▌                                                                                     | 921/3996 [1:08:53<3:37:01,  4.23s/it]
 23%|█████████████████████████▌                                                                                     | 922/3996 [1:08:58<3:44:53,  4.39s/it]
 23%|█████████████████████████▋                                                                                     | 923/3996 [1:09:02<3:42:22,  4.34s/it]
 23%|█████████████████████████▋                                                                                     | 924/3996 [1:09:06<3:40:13,  4.30s/it]
 23%|█████████████████████████▋                                                                                     | 925/3996 [1:09:10<3:38:35,  4.27s/it]
                                                                                                                                                           
{'loss': 0.5511, 'grad_norm': 0.1948954463005066, 'learning_rate': 0.0001787259982191692, 'ppl': 1.7352, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4237.98, 'total_tokens': 17841287, 'epoch': 0.69}
+
 23%|█████████████████████████▋                                                                                     | 925/3996 [1:09:10<3:38:35,  4.27s/it]
 23%|█████████████████████████▋                                                                                     | 926/3996 [1:09:15<3:37:32,  4.25s/it]
 23%|█████████████████████████▊                                                                                     | 927/3996 [1:09:19<3:36:59,  4.24s/it]
 23%|█████████████████████████▊                                                                                     | 928/3996 [1:09:23<3:36:47,  4.24s/it]
 23%|█████████████████████████▊                                                                                     | 929/3996 [1:09:28<3:44:38,  4.39s/it]
 23%|█████████████████████████▊                                                                                     | 930/3996 [1:09:32<3:42:12,  4.35s/it]
 23%|█████████████████████████▊                                                                                     | 931/3996 [1:09:36<3:39:54,  4.30s/it]
 23%|█████████████████████████▉                                                                                     | 932/3996 [1:09:40<3:38:19,  4.28s/it]
 23%|█████████████████████████▉                                                                                     | 933/3996 [1:09:45<3:37:05,  4.25s/it]
 23%|█████████████████████████▉                                                                                     | 934/3996 [1:09:49<3:36:14,  4.24s/it]
 23%|█████████████████████████▉                                                                                     | 935/3996 [1:09:53<3:35:57,  4.23s/it]
 23%|██████████████████████████                                                                                     | 936/3996 [1:09:58<3:43:47,  4.39s/it]
 23%|██████████████████████████                                                                                     | 937/3996 [1:10:02<3:41:12,  4.34s/it]
 23%|██████████████████████████                                                                                     | 938/3996 [1:10:06<3:39:00,  4.30s/it]
 23%|██████████████████████████                                                                                     | 939/3996 [1:10:10<3:37:27,  4.27s/it]
 24%|██████████████████████████                                                                                     | 940/3996 [1:10:15<3:36:37,  4.25s/it]
 24%|██████████████████████████▏                                                                                    | 941/3996 [1:10:19<3:36:00,  4.24s/it]
 24%|██████████████████████████▏                                                                                    | 942/3996 [1:10:23<3:35:17,  4.23s/it]
 24%|██████████████████████████▏                                                                                    | 943/3996 [1:10:28<3:42:57,  4.38s/it]
 24%|██████████████████████████▏                                                                                    | 944/3996 [1:10:32<3:40:29,  4.33s/it]
 24%|██████████████████████████▎                                                                                    | 945/3996 [1:10:36<3:38:33,  4.30s/it]
 24%|██████████████████████████▎                                                                                    | 946/3996 [1:10:40<3:37:16,  4.27s/it]
 24%|██████████████████████████▎                                                                                    | 947/3996 [1:10:45<3:36:08,  4.25s/it]
 24%|██████████████████████████▎                                                                                    | 948/3996 [1:10:49<3:35:22,  4.24s/it]
 24%|██████████████████████████▎                                                                                    | 949/3996 [1:10:53<3:34:42,  4.23s/it]
 24%|██████████████████████████▍                                                                                    | 950/3996 [1:10:58<3:42:35,  4.38s/it]
                                                                                                                                                           
{'loss': 0.5532, 'grad_norm': 0.19541053473949432, 'learning_rate': 0.00017746703398601872, 'ppl': 1.7388, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3725.33, 'total_tokens': 18283596, 'epoch': 0.71}
+
 24%|██████████████████████████▍                                                                                    | 950/3996 [1:10:58<3:42:35,  4.38s/it]
 24%|██████████████████████████▍                                                                                    | 951/3996 [1:11:02<3:39:49,  4.33s/it]
 24%|██████████████████████████▍                                                                                    | 952/3996 [1:11:06<3:37:42,  4.29s/it]
 24%|██████████████████████████▍                                                                                    | 953/3996 [1:11:10<3:36:02,  4.26s/it]
 24%|██████████████████████████▌                                                                                    | 954/3996 [1:11:15<3:35:10,  4.24s/it]
 24%|██████████████████████████▌                                                                                    | 955/3996 [1:11:19<3:34:27,  4.23s/it]
 24%|██████████████████████████▌                                                                                    | 956/3996 [1:11:23<3:34:00,  4.22s/it]
 24%|██████████████████████████▌                                                                                    | 957/3996 [1:11:28<3:41:31,  4.37s/it]
 24%|██████████████████████████▌                                                                                    | 958/3996 [1:11:32<3:38:49,  4.32s/it]
 24%|██████████████████████████▋                                                                                    | 959/3996 [1:11:36<3:36:42,  4.28s/it]
 24%|██████████████████████████▋                                                                                    | 960/3996 [1:11:40<3:35:23,  4.26s/it]
 24%|██████████████████████████▋                                                                                    | 961/3996 [1:11:44<3:34:42,  4.24s/it]
 24%|██████████████████████████▋                                                                                    | 962/3996 [1:11:49<3:34:12,  4.24s/it]
 24%|██████████████████████████▊                                                                                    | 963/3996 [1:11:53<3:33:35,  4.23s/it]
 24%|██████████████████████████▊                                                                                    | 964/3996 [1:11:58<3:48:17,  4.52s/it]
 24%|██████████████████████████▊                                                                                    | 965/3996 [1:12:02<3:43:25,  4.42s/it]
 24%|██████████████████████████▊                                                                                    | 966/3996 [1:12:07<3:40:05,  4.36s/it]
 24%|██████████████████████████▊                                                                                    | 967/3996 [1:12:11<3:37:33,  4.31s/it]
 24%|██████████████████████████▉                                                                                    | 968/3996 [1:12:15<3:35:58,  4.28s/it]
 24%|██████████████████████████▉                                                                                    | 969/3996 [1:12:19<3:34:36,  4.25s/it]
 24%|██████████████████████████▉                                                                                    | 970/3996 [1:12:23<3:33:52,  4.24s/it]
 24%|██████████████████████████▉                                                                                    | 971/3996 [1:12:28<3:41:11,  4.39s/it]
 24%|███████████████████████████                                                                                    | 972/3996 [1:12:32<3:38:31,  4.34s/it]
 24%|███████████████████████████                                                                                    | 973/3996 [1:12:36<3:36:30,  4.30s/it]
 24%|███████████████████████████                                                                                    | 974/3996 [1:12:41<3:34:57,  4.27s/it]
 24%|███████████████████████████                                                                                    | 975/3996 [1:12:45<3:34:00,  4.25s/it]
                                                                                                                                                           
{'loss': 0.5521, 'grad_norm': 0.1818365603685379, 'learning_rate': 0.0001761765890741701, 'ppl': 1.7369, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4211.63, 'total_tokens': 18726722, 'epoch': 0.73}
+
 24%|███████████████████████████                                                                                    | 975/3996 [1:12:45<3:34:00,  4.25s/it]
 24%|███████████████████████████                                                                                    | 976/3996 [1:12:49<3:33:24,  4.24s/it]
 24%|███████████████████████████▏                                                                                   | 977/3996 [1:12:53<3:32:58,  4.23s/it]
 24%|███████████████████████████▏                                                                                   | 978/3996 [1:12:58<3:40:35,  4.39s/it]
 24%|███████████████████████████▏                                                                                   | 979/3996 [1:13:02<3:37:49,  4.33s/it]
 25%|███████████████████████████▏                                                                                   | 980/3996 [1:13:06<3:35:34,  4.29s/it]
 25%|███████████████████████████▎                                                                                   | 981/3996 [1:13:11<3:34:06,  4.26s/it]
 25%|███████████████████████████▎                                                                                   | 982/3996 [1:13:15<3:33:27,  4.25s/it]
 25%|███████████████████████████▎                                                                                   | 983/3996 [1:13:19<3:32:49,  4.24s/it]
 25%|███████████████████████████▎                                                                                   | 984/3996 [1:13:23<3:32:13,  4.23s/it]
 25%|███████████████████████████▎                                                                                   | 985/3996 [1:13:28<3:39:43,  4.38s/it]
 25%|███████████████████████████▍                                                                                   | 986/3996 [1:13:32<3:37:05,  4.33s/it]
 25%|███████████████████████████▍                                                                                   | 987/3996 [1:13:36<3:35:08,  4.29s/it]
 25%|███████████████████████████▍                                                                                   | 988/3996 [1:13:41<3:33:56,  4.27s/it]
 25%|███████████████████████████▍                                                                                   | 989/3996 [1:13:45<3:32:40,  4.24s/it]
 25%|███████████████████████████▌                                                                                   | 990/3996 [1:13:49<3:32:00,  4.23s/it]
 25%|███████████████████████████▌                                                                                   | 991/3996 [1:13:53<3:31:42,  4.23s/it]
 25%|███████████████████████████▌                                                                                   | 992/3996 [1:13:58<3:39:14,  4.38s/it]
 25%|███████████████████████████▌                                                                                   | 993/3996 [1:14:02<3:36:36,  4.33s/it]
 25%|███████████████████████████▌                                                                                   | 994/3996 [1:14:06<3:34:34,  4.29s/it]
 25%|███████████████████████████▋                                                                                   | 995/3996 [1:14:11<3:33:20,  4.27s/it]
 25%|███████████████████████████▋                                                                                   | 996/3996 [1:14:15<3:32:41,  4.25s/it]
 25%|███████████████████████████▋                                                                                   | 997/3996 [1:14:19<3:31:54,  4.24s/it]
 25%|███████████████████████████▋                                                                                   | 998/3996 [1:14:23<3:31:33,  4.23s/it]
 25%|███████████████████████████▊                                                                                   | 999/3996 [1:14:28<3:39:12,  4.39s/it]
 25%|███████████████████████████▌                                                                                  | 1000/3996 [1:14:32<3:40:01,  4.41s/it]
                                                                                                                                                           
{'loss': 0.5511, 'grad_norm': 0.1838025599718094, 'learning_rate': 0.00017485518788838705, 'ppl': 1.7352, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3962.4, 'total_tokens': 19167258, 'epoch': 0.75}
+
 25%|███████████████████████████▌                                                                                  | 1000/3996 [1:14:32<3:40:01,  4.41s/it][2025-12-29 04:04:18,203] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:3751] Running evaluation step...
+[2025-12-29 04:04:19,132] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.44579172134399414
+[2025-12-29 04:04:19,560] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.42780184745788574
+[2025-12-29 04:04:19,955] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.3948495388031006
+[2025-12-29 04:04:20,352] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.3962395191192627
+[2025-12-29 04:04:20,352] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:3751] gather_len_batches: [72]
 
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.41it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:54,  1.59it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<00:59,  1.43it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:05,  1.30it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:05,  1.28it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:08,  1.21it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:06<01:07,  1.22it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:08,  1.19it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:06,  1.20it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:07,  1.17it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:05,  1.19it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:06,  1.17it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:04,  1.18it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:13,  1.02it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:13<01:06,  1.12it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:14<01:06,  1.10it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:03,  1.14it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:03,  1.13it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<01:00,  1.16it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<01:00,  1.14it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:58,  1.16it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:19<00:58,  1.14it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:20<00:56,  1.17it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:56,  1.15it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.17it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:54,  1.15it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:52,  1.17it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:52,  1.15it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:25<00:51,  1.17it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:26<00:51,  1.15it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.17it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:49,  1.15it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:47,  1.17it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:47,  1.15it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:45,  1.18it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:46,  1.15it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:32<00:44,  1.17it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:33<00:44,  1.14it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:43,  1.16it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:42,  1.14it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:41,  1.16it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:41,  1.14it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:39,  1.16it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:38<00:39,  1.14it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:39<00:37,  1.17it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.13it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:36,  1.16it/s]
-
 54%|█████████████████████��██████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.14it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:34,  1.17it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:33,  1.15it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:44<00:32,  1.17it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:45<00:32,  1.15it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.17it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.14it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:29,  1.17it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.15it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.17it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:26,  1.15it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:51<00:25,  1.17it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:52<00:25,  1.15it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:23,  1.17it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:54<00:27,  1.01s/it]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:23,  1.12it/s]
-
 72%|███████████████████████████████████████████████████████████████████████████��█████████▏                                | 65/90 [00:55<00:22,  1.12it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:20,  1.15it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:57<00:20,  1.14it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:58<00:18,  1.16it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:59<00:18,  1.15it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.17it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.15it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.17it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.15it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.17it/s]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:04<00:12,  1.15it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:05<00:11,  1.18it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:06<00:11,  1.15it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.17it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.15it/s]
-
 89%|█████████████████████████████���██████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.17it/s]
-
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 81/90 [01:09<00:07,  1.15it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:10<00:06,  1.17it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:11<00:06,  1.14it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:12<00:05,  1.18it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.13it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.17it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.13it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:15<00:01,  1.17it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89/90 [01:16<00:00,  1.13it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.5492991805076599, 'eval_runtime': 79.4635, 'eval_samples_per_second': 9.187, 'eval_steps_per_second': 2.303, 'eval_ppl': 1.732, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.17, 'epoch': 0.72}
-
 24%|██████████████████████████▋                                                                                    | 600/2499 [1:12:50<3:18:31,  6.27s/it]
-
100%|███████████████████████████████████████████████████████████��██████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
-
                                                                                                                                                           [2025-12-28 12:18:30,249] [INFO] [axolotl.core.trainers.base._save:692] [PID:42410] Saving model checkpoint to ./outputs/luau-codellama-h200/checkpoint-600
-
 24%|██████████████████████████▍                                                                                   | 601/2499 [1:13:01<17:40:22, 33.52s/it]
                                                                                                                                                           
{'loss': 0.5431, 'grad_norm': 0.15443255007266998, 'learning_rate': 0.00017352963349382875, 'ppl': 1.7213, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.05, 'tokens_per_second_per_gpu': 4740.78, 'total_tokens': 26107242, 'epoch': 0.72}
-
 24%|██████████████████████████▍                                                                                   | 601/2499 [1:13:01<17:40:22, 33.52s/it]
 24%|██████████████████████████▍                                                                                   | 602/2499 [1:13:07<13:21:04, 25.34s/it]
                                                                                                                                                           
{'loss': 0.5608, 'grad_norm': 0.15965475142002106, 'learning_rate': 0.00017344403050746084, 'ppl': 1.7521, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.05, 'tokens_per_second_per_gpu': 4474.57, 'total_tokens': 26135160, 'epoch': 0.72}
-
 24%|██████████████████████████▍                                                                                   | 602/2499 [1:13:07<13:21:04, 25.34s/it]
 24%|██████████████████████████▌                                                                                   | 603/2499 [1:13:13<10:19:58, 19.62s/it]
                                                                                                                                                           
{'loss': 0.5405, 'grad_norm': 0.15331172943115234, 'learning_rate': 0.00017335831051556064, 'ppl': 1.7169, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4560.89, 'total_tokens': 26163788, 'epoch': 0.72}
-
 24%|██████████████████████████▌                                                                                   | 603/2499 [1:13:13<10:19:58, 19.62s/it]
 24%|██████████████████████████▊                                                                                    | 604/2499 [1:13:20<8:13:08, 15.61s/it]
                                                                                                                                                           
{'loss': 0.554, 'grad_norm': 0.15818923711776733, 'learning_rate': 0.00017327247365469078, 'ppl': 1.7402, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4679.66, 'total_tokens': 26193104, 'epoch': 0.73}
-
 24%|██████████████████████████▊                                                                                    | 604/2499 [1:13:20<8:13:08, 15.61s/it]
 24%|██████████████████████████▊                                                                                    | 605/2499 [1:13:26<6:44:32, 12.82s/it]
                                                                                                                                                           
{'loss': 0.5676, 'grad_norm': 0.1669849008321762, 'learning_rate': 0.0001731865200616001, 'ppl': 1.764, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4402.46, 'total_tokens': 26220768, 'epoch': 0.73}
-
 24%|██████████████████████████▊                                                                                    | 605/2499 [1:13:26<6:44:32, 12.82s/it]
 24%|██████████████████████████▉                                                                                    | 606/2499 [1:13:32<5:42:05, 10.84s/it]
                                                                                                                                                           
{'loss': 0.5567, 'grad_norm': 0.1447984129190445, 'learning_rate': 0.00017310044987322348, 'ppl': 1.7449, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4433.92, 'total_tokens': 26248418, 'epoch': 0.73}
-
 24%|██████████████████████████▉                                                                                    | 606/2499 [1:13:32<5:42:05, 10.84s/it]
 24%|██████████████████████████▉                                                                                    | 607/2499 [1:13:38<4:58:29,  9.47s/it]
                                                                                                                                                           
{'loss': 0.6084, 'grad_norm': 0.17044642567634583, 'learning_rate': 0.00017301426322668143, 'ppl': 1.8375, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4454.73, 'total_tokens': 26276263, 'epoch': 0.73}
-
 24%|██████████████████████████▉                                                                                    | 607/2499 [1:13:38<4:58:29,  9.47s/it]
 24%|███████████████████████████                                                                                    | 608/2499 [1:13:45<4:28:09,  8.51s/it]
                                                                                                                                                           
{'loss': 0.5733, 'grad_norm': 0.1568867266178131, 'learning_rate': 0.00017292796025928012, 'ppl': 1.7741, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4658.56, 'total_tokens': 26305482, 'epoch': 0.73}
-
 24%|███████████████████████████                                                                                    | 608/2499 [1:13:45<4:28:09,  8.51s/it]
 24%|███████████████████████████                                                                                    | 609/2499 [1:13:51<4:06:48,  7.84s/it]
                                                                                                                                                           
{'loss': 0.586, 'grad_norm': 0.155142679810524, 'learning_rate': 0.000172841541108511, 'ppl': 1.7968, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4641.87, 'total_tokens': 26334550, 'epoch': 0.73}
-
 24%|███████████████████████████                                                                                    | 609/2499 [1:13:51<4:06:48,  7.84s/it]
 24%|███████████████████████████                                                                                    | 610/2499 [1:13:57<3:51:45,  7.36s/it]
                                                                                                                                                           
{'loss': 0.5379, 'grad_norm': 0.164072185754776, 'learning_rate': 0.00017275500591205055, 'ppl': 1.7124, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4624.75, 'total_tokens': 26363473, 'epoch': 0.73}
-
 24%|███████████████████████████                                                                                    | 610/2499 [1:13:57<3:51:45,  7.36s/it]
 24%|███████████████████████████▏                                                                                   | 611/2499 [1:14:04<3:41:31,  7.04s/it]
                                                                                                                                                           
{'loss': 0.5643, 'grad_norm': 0.14827710390090942, 'learning_rate': 0.00017266835480776014, 'ppl': 1.7582, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4589.99, 'total_tokens': 26392329, 'epoch': 0.73}
-
 24%|███████████████████████████▏                                                                                   | 611/2499 [1:14:04<3:41:31,  7.04s/it]
 24%|███████████████████████████▏                                                                                   | 612/2499 [1:14:10<3:33:57,  6.80s/it]
                                                                                                                                                           
{'loss': 0.5551, 'grad_norm': 0.14518024027347565, 'learning_rate': 0.0001725815879336859, 'ppl': 1.7421, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4454.43, 'total_tokens': 26420158, 'epoch': 0.73}
-
 24%|███████████████████████████▏                                                                                   | 612/2499 [1:14:10<3:33:57,  6.80s/it]
 25%|███████████████████████████▏                                                                                   | 613/2499 [1:14:16<3:28:27,  6.63s/it]
                                                                                                                                                           
{'loss': 0.5249, 'grad_norm': 0.14937075972557068, 'learning_rate': 0.00017249470542805826, 'ppl': 1.6903, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4379.43, 'total_tokens': 26447441, 'epoch': 0.74}
-
 25%|███████████████████████████▏                                                                                   | 613/2499 [1:14:16<3:28:27,  6.63s/it]
 25%|███████████████████████████▎                                                                                   | 614/2499 [1:14:22<3:24:56,  6.52s/it]
                                                                                                                                                           
{'loss': 0.5378, 'grad_norm': 0.1505361795425415, 'learning_rate': 0.00017240770742929192, 'ppl': 1.7122, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.07, 'tokens_per_second_per_gpu': 4620.94, 'total_tokens': 26476403, 'epoch': 0.74}
-
 25%|███████████████████████████▎                                                                                   | 614/2499 [1:14:22<3:24:56,  6.52s/it]
 25%|███████████████████████████▎                                                                                   | 615/2499 [1:14:29<3:22:14,  6.44s/it]
                                                                                                                                                           
{'loss': 0.557, 'grad_norm': 0.15796837210655212, 'learning_rate': 0.00017232059407598565, 'ppl': 1.7454, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4509.85, 'total_tokens': 26504572, 'epoch': 0.74}
-
 25%|███████████████████████████▎                                                                                   | 615/2499 [1:14:29<3:22:14,  6.44s/it]
 25%|███████████████████████████▎                                                                                   | 616/2499 [1:14:35<3:20:16,  6.38s/it]
                                                                                                                                                           
{'loss': 0.5542, 'grad_norm': 0.22552503645420074, 'learning_rate': 0.00017223336550692186, 'ppl': 1.7405, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4382.77, 'total_tokens': 26531925, 'epoch': 0.74}
-
 25%|███████████████████████████▎                                                                                   | 616/2499 [1:14:35<3:20:16,  6.38s/it]
 25%|███████████████████████████▍                                                                                   | 617/2499 [1:14:41<3:18:54,  6.34s/it]
                                                                                                                                                           
{'loss': 0.5768, 'grad_norm': 0.15645365417003632, 'learning_rate': 0.00017214602186106662, 'ppl': 1.7803, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4453.04, 'total_tokens': 26559739, 'epoch': 0.74}
-
 25%|███████████████████████████▍                                                                                   | 617/2499 [1:14:41<3:18:54,  6.34s/it]
 25%|███████████████████████████▍                                                                                   | 618/2499 [1:14:47<3:18:18,  6.33s/it]
                                                                                                                                                           
{'loss': 0.5381, 'grad_norm': 0.14894016087055206, 'learning_rate': 0.00017205856327756925, 'ppl': 1.7127, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4469.12, 'total_tokens': 26587835, 'epoch': 0.74}
-
 25%|███████████████████████████▍                                                                                   | 618/2499 [1:14:47<3:18:18,  6.33s/it]
 25%|███████████████████████████▍                                                                                   | 619/2499 [1:14:54<3:17:25,  6.30s/it]
                                                                                                                                                           
{'loss': 0.5574, 'grad_norm': 0.16387908160686493, 'learning_rate': 0.00017197098989576222, 'ppl': 1.7461, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4446.37, 'total_tokens': 26615575, 'epoch': 0.74}
-
 25%|███████████████████████████▍                                                                                   | 619/2499 [1:14:54<3:17:25,  6.30s/it]
 25%|███████████████████████████▌                                                                                   | 620/2499 [1:15:00<3:16:50,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5345, 'grad_norm': 0.1502378284931183, 'learning_rate': 0.00017188330185516094, 'ppl': 1.7066, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4413.85, 'total_tokens': 26643152, 'epoch': 0.74}
-
 25%|███████████████████████████▌                                                                                   | 620/2499 [1:15:00<3:16:50,  6.29s/it]
 25%|███████████████████████████▌                                                                                   | 621/2499 [1:15:06<3:16:25,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5992, 'grad_norm': 0.1670679748058319, 'learning_rate': 0.00017179549929546335, 'ppl': 1.8207, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4508.66, 'total_tokens': 26671327, 'epoch': 0.75}
-
 25%|███████████████████████████▌                                                                                   | 621/2499 [1:15:06<3:16:25,  6.28s/it]
 25%|███████████████████████████▋                                                                                   | 622/2499 [1:15:12<3:16:10,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5946, 'grad_norm': 0.16181008517742157, 'learning_rate': 0.00017170758235654997, 'ppl': 1.8123, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4577.69, 'total_tokens': 26699973, 'epoch': 0.75}
-
 25%|███████████████████████████▋                                                                                   | 622/2499 [1:15:12<3:16:10,  6.27s/it]
 25%|███████████████████████████▋                                                                                   | 623/2499 [1:15:19<3:16:02,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5877, 'grad_norm': 0.17699268460273743, 'learning_rate': 0.0001716195511784835, 'ppl': 1.7998, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4582.31, 'total_tokens': 26728687, 'epoch': 0.75}
-
 25%|███████��███████████████████▋                                                                                   | 623/2499 [1:15:19<3:16:02,  6.27s/it]
 25%|███████████████████████████▋                                                                                   | 624/2499 [1:15:25<3:16:15,  6.28s/it]
                                                                                                                                                           
{'loss': 0.546, 'grad_norm': 0.16804426908493042, 'learning_rate': 0.0001715314059015086, 'ppl': 1.7263, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4576.77, 'total_tokens': 26757523, 'epoch': 0.75}
-
 25%|███████████████████████████▋                                                                                   | 624/2499 [1:15:25<3:16:15,  6.28s/it]
 25%|███████████████████████████▊                                                                                   | 625/2499 [1:15:31<3:16:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5654, 'grad_norm': 0.1552819013595581, 'learning_rate': 0.00017144314666605172, 'ppl': 1.7602, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4491.13, 'total_tokens': 26785650, 'epoch': 0.75}
-
 25%|███████████████████████████▊                                                                                   | 625/2499 [1:15:31<3:16:01,  6.28s/it]
 25%|███████████████████████████▊                                                                                   | 626/2499 [1:15:37<3:15:33,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5238, 'grad_norm': 0.16035959124565125, 'learning_rate': 0.0001713547736127209, 'ppl': 1.6884, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4372.25, 'total_tokens': 26812912, 'epoch': 0.75}
-
 25%|███████████████████████████▊                                                                                   | 626/2499 [1:15:37<3:15:33,  6.26s/it]
 25%|███████████████████████████▊                                                                                   | 627/2499 [1:15:44<3:15:27,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5031, 'grad_norm': 0.15231968462467194, 'learning_rate': 0.00017126628688230545, 'ppl': 1.6538, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4624.25, 'total_tokens': 26841872, 'epoch': 0.75}
-
 25%|███████████████████████████▊                                                                                   | 627/2499 [1:15:44<3:15:27,  6.26s/it]
 25%|███████████████████████████▉                                                                                   | 628/2499 [1:15:50<3:15:22,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5672, 'grad_norm': 0.1670321822166443, 'learning_rate': 0.0001711776866157758, 'ppl': 1.7633, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4552.49, 'total_tokens': 26870393, 'epoch': 0.75}
-
 25%|███████████████████████████▉                                                                                   | 628/2499 [1:15:50<3:15:22,  6.27s/it]
 25%|███████████████████████████▉                                                                                   | 629/2499 [1:15:56<3:15:00,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5665, 'grad_norm': 0.16463960707187653, 'learning_rate': 0.00017108897295428326, 'ppl': 1.7621, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4309.71, 'total_tokens': 26897258, 'epoch': 0.76}
-
 25%|███████████████████████████▉                                                                                   | 629/2499 [1:15:56<3:15:00,  6.26s/it]
 25%|███████████████████████████▉                                                                                   | 630/2499 [1:16:02<3:15:03,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5288, 'grad_norm': 0.15193606913089752, 'learning_rate': 0.0001710001460391598, 'ppl': 1.6969, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4405.36, 'total_tokens': 26924890, 'epoch': 0.76}
-
 25%|███████████████████████████▉                                                                                   | 630/2499 [1:16:02<3:15:03,  6.26s/it]
 25%|████████████████████████████                                                                                   | 631/2499 [1:16:09<3:14:50,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5362, 'grad_norm': 0.16677305102348328, 'learning_rate': 0.00017091120601191786, 'ppl': 1.7095, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4439.72, 'total_tokens': 26952621, 'epoch': 0.76}
-
 25%|████████████████████████████                                                                                   | 631/2499 [1:16:09<3:14:50,  6.26s/it]
 25%|████████████████████████████                                                                                   | 632/2499 [1:16:15<3:14:45,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5676, 'grad_norm': 0.17070624232292175, 'learning_rate': 0.00017082215301424998, 'ppl': 1.764, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4575.59, 'total_tokens': 26981256, 'epoch': 0.76}
-
 25%|████████████████████████████                                                                                   | 632/2499 [1:16:15<3:14:45,  6.26s/it]
 25%|████████████████████████████                                                                                   | 633/2499 [1:16:21<3:14:32,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5367, 'grad_norm': 0.15856873989105225, 'learning_rate': 0.00017073298718802871, 'ppl': 1.7104, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4297.55, 'total_tokens': 27008087, 'epoch': 0.76}
-
 25%|████████████████████████████                                                                                   | 633/2499 [1:16:21<3:14:32,  6.26s/it]
 25%|████████████████████████████▏                                                                                  | 634/2499 [1:16:27<3:14:24,  6.25s/it]
                                                                                                                                                           
{'loss': 0.5403, 'grad_norm': 0.15367814898490906, 'learning_rate': 0.00017064370867530645, 'ppl': 1.7165, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4376.93, 'total_tokens': 27035441, 'epoch': 0.76}
-
 25%|████████████████████████████▏                                                                                  | 634/2499 [1:16:27<3:14:24,  6.25s/it]
 25%|████████████████████████████▏                                                                                  | 635/2499 [1:16:34<3:14:11,  6.25s/it]
                                                                                                                                                           
{'loss': 0.6084, 'grad_norm': 0.1730221062898636, 'learning_rate': 0.00017055431761831498, 'ppl': 1.8375, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4387.66, 'total_tokens': 27062820, 'epoch': 0.76}
-
 25%|████████████████████████████▏                                                                                  | 635/2499 [1:16:34<3:14:11,  6.25s/it]
 25%|████████████████████████████▏                                                                                  | 636/2499 [1:16:40<3:14:27,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5498, 'grad_norm': 0.165005624294281, 'learning_rate': 0.00017046481415946549, 'ppl': 1.7329, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4582.61, 'total_tokens': 27091639, 'epoch': 0.76}
-
 25%|████████████████████████████▏                                                                                  | 636/2499 [1:16:40<3:14:27,  6.26s/it]
 25%|████████████████████████████▎                                                                                  | 637/2499 [1:16:46<3:14:39,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5798, 'grad_norm': 0.16355308890342712, 'learning_rate': 0.00017037519844134813, 'ppl': 1.7857, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4530.74, 'total_tokens': 27120147, 'epoch': 0.76}
-
 25%|████████████████████████████▎                                                                                  | 637/2499 [1:16:46<3:14:39,  6.27s/it]
 26%|████████████████████████████▎                                                                                  | 638/2499 [1:16:53<3:14:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6201, 'grad_norm': 0.15821842849254608, 'learning_rate': 0.000170285470606732, 'ppl': 1.8591, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4515.18, 'total_tokens': 27148403, 'epoch': 0.77}
-
 26%|████████████████████████████▎                                                                                  | 638/2499 [1:16:53<3:14:26,  6.27s/it]
 26%|████████████████████████████▍                                                                                  | 639/2499 [1:16:59<3:14:06,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5461, 'grad_norm': 0.15199202299118042, 'learning_rate': 0.00017019563079856474, 'ppl': 1.7265, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4300.16, 'total_tokens': 27175240, 'epoch': 0.77}
-
 26%|████████████████████████████▍                                                                                  | 639/2499 [1:16:59<3:14:06,  6.26s/it]
 26%|████████████████████████████▍                                                                                  | 640/2499 [1:17:05<3:13:55,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5573, 'grad_norm': 0.15636083483695984, 'learning_rate': 0.00017010567915997244, 'ppl': 1.746, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4436.65, 'total_tokens': 27202969, 'epoch': 0.77}
-
 26%|████████████████████████████▍                                                                                  | 640/2499 [1:17:05<3:13:55,  6.26s/it]
 26%|████████████████████████████▍                                                                                  | 641/2499 [1:17:11<3:13:42,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5442, 'grad_norm': 0.14868567883968353, 'learning_rate': 0.00017001561583425932, 'ppl': 1.7232, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4426.99, 'total_tokens': 27230617, 'epoch': 0.77}
-
 26%|████████████████████████████▍                                                                                  | 641/2499 [1:17:11<3:13:42,  6.26s/it]
 26%|████████████████████████████▌                                                                                  | 642/2499 [1:17:18<3:13:37,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5611, 'grad_norm': 0.1626492142677307, 'learning_rate': 0.0001699254409649075, 'ppl': 1.7526, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4496.31, 'total_tokens': 27258739, 'epoch': 0.77}
-
 26%|████████████████████████████▌                                                                                  | 642/2499 [1:17:18<3:13:37,  6.26s/it]
 26%|████████████████████████████▌                                                                                  | 643/2499 [1:17:24<3:13:48,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5445, 'grad_norm': 0.15078237652778625, 'learning_rate': 0.00016983515469557684, 'ppl': 1.7237, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4534.17, 'total_tokens': 27287235, 'epoch': 0.77}
-
 26%|████████████████████████████▌                                                                                  | 643/2499 [1:17:24<3:13:48,  6.27s/it]
 26%|████████████████████████████▌                                                                                  | 644/2499 [1:17:30<3:13:41,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5867, 'grad_norm': 0.1609424501657486, 'learning_rate': 0.00016974475717010468, 'ppl': 1.798, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4581.26, 'total_tokens': 27315915, 'epoch': 0.77}
-
 26%|████████████████████████████▌                                                                                  | 644/2499 [1:17:30<3:13:41,  6.26s/it]
 26%|████████████████████████████▋                                                                                  | 645/2499 [1:17:36<3:13:27,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5694, 'grad_norm': 0.1608027219772339, 'learning_rate': 0.00016965424853250557, 'ppl': 1.7672, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4355.39, 'total_tokens': 27343127, 'epoch': 0.77}
-
 26%|████████████████████████████▋                                                                                  | 645/2499 [1:17:36<3:13:27,  6.26s/it]
 26%|████████████████████████████▋                                                                                  | 646/2499 [1:17:43<3:13:27,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5648, 'grad_norm': 0.15912018716335297, 'learning_rate': 0.00016956362892697112, 'ppl': 1.7591, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4732.01, 'total_tokens': 27372800, 'epoch': 0.78}
-
 26%|████████████████████████████▋                                                                                  | 646/2499 [1:17:43<3:13:27,  6.26s/it]
 26%|█████████████████████���██████▋                                                                                  | 647/2499 [1:17:49<3:13:21,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5446, 'grad_norm': 0.16331568360328674, 'learning_rate': 0.00016947289849786974, 'ppl': 1.7239, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4598.22, 'total_tokens': 27401590, 'epoch': 0.78}
-
 26%|████████████████████████████▋                                                                                  | 647/2499 [1:17:49<3:13:21,  6.26s/it]
 26%|████████████████████████████▊                                                                                  | 648/2499 [1:17:55<3:13:14,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5874, 'grad_norm': 0.16428092122077942, 'learning_rate': 0.00016938205738974626, 'ppl': 1.7993, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4608.8, 'total_tokens': 27430448, 'epoch': 0.78}
-
 26%|████████████████████████████▊                                                                                  | 648/2499 [1:17:55<3:13:14,  6.26s/it]
 26%|████████████████████████████▊                                                                                  | 649/2499 [1:18:01<3:13:11,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5681, 'grad_norm': 0.1655767410993576, 'learning_rate': 0.00016929110574732202, 'ppl': 1.7649, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4383.76, 'total_tokens': 27457921, 'epoch': 0.78}
-
 26%|████████████████████████████▊                                                                                  | 649/2499 [1:18:01<3:13:11,  6.27s/it]
 26%|████████████████████████████▊                                                                                  | 650/2499 [1:18:08<3:13:14,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5865, 'grad_norm': 0.15909960865974426, 'learning_rate': 0.0001692000437154943, 'ppl': 1.7977, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4610.29, 'total_tokens': 27486869, 'epoch': 0.78}
-
 26%|████████████████████████████▊                                                                                  | 650/2499 [1:18:08<3:13:14,  6.27s/it]
 26%|████████████████████████████▉                                                                                  | 651/2499 [1:18:14<3:13:06,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5777, 'grad_norm': 0.1619177907705307, 'learning_rate': 0.00016910887143933636, 'ppl': 1.7819, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4469.57, 'total_tokens': 27514869, 'epoch': 0.78}
-
 26%|████████████████████████████▉                                                                                  | 651/2499 [1:18:14<3:13:06,  6.27s/it]
 26%|████████████████████████████▉                                                                                  | 652/2499 [1:18:20<3:12:57,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5718, 'grad_norm': 0.1830313801765442, 'learning_rate': 0.00016901758906409705, 'ppl': 1.7715, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4500.34, 'total_tokens': 27543048, 'epoch': 0.78}
-
 26%|████████████████████████████▉                                                                                  | 652/2499 [1:18:20<3:12:57,  6.27s/it]
 26%|█████████████████████████████                                                                                  | 653/2499 [1:18:26<3:13:02,  6.27s/it]
                                                                                                                                                           
{'loss': 0.551, 'grad_norm': 0.16313879191875458, 'learning_rate': 0.00016892619673520057, 'ppl': 1.735, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4727.61, 'total_tokens': 27572770, 'epoch': 0.78}
-
 26%|█████████████████████████████                                                                                  | 653/2499 [1:18:26<3:13:02,  6.27s/it]
 26%|█████████████████████████████                                                                                  | 654/2499 [1:18:33<3:12:58,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5615, 'grad_norm': 0.15075667202472687, 'learning_rate': 0.00016883469459824644, 'ppl': 1.7533, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4654.0, 'total_tokens': 27601981, 'epoch': 0.79}
-
 26%|█████████████████████████████                                                                                  | 654/2499 [1:18:33<3:12:58,  6.28s/it]
 26%|█████████████████████████████                                                                                  | 655/2499 [1:18:39<3:13:06,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5551, 'grad_norm': 0.15292450785636902, 'learning_rate': 0.0001687430827990089, 'ppl': 1.7421, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4633.85, 'total_tokens': 27631169, 'epoch': 0.79}
-
 26%|█████████████████████████████                                                                                  | 655/2499 [1:18:39<3:13:06,  6.28s/it]
 26%|█████████████████████████████▏                                                                                 | 656/2499 [1:18:45<3:12:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6184, 'grad_norm': 0.1681700199842453, 'learning_rate': 0.00016865136148343706, 'ppl': 1.856, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4542.76, 'total_tokens': 27659667, 'epoch': 0.79}
-
 26%|█████████████████████████████▏                                                                                 | 656/2499 [1:18:45<3:12:57,  6.28s/it]
 26%|█████████████████████████████▏                                                                                 | 657/2499 [1:18:52<3:12:37,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5921, 'grad_norm': 0.16229918599128723, 'learning_rate': 0.00016855953079765448, 'ppl': 1.8078, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4427.98, 'total_tokens': 27687361, 'epoch': 0.79}
-
 26%|█████████████████████████████▏                                                                                 | 657/2499 [1:18:52<3:12:37,  6.27s/it]
 26%|█████████████████████████████▏                                                                                 | 658/2499 [1:18:58<3:12:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5735, 'grad_norm': 0.15574562549591064, 'learning_rate': 0.0001684675908879589, 'ppl': 1.7745, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4419.82, 'total_tokens': 27715048, 'epoch': 0.79}
-
 26%|█████████████████████████████▏                                                                                 | 658/2499 [1:18:58<3:12:26,  6.27s/it]
 26%|█████████████████████████████▎                                                                                 | 659/2499 [1:19:04<3:12:24,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5574, 'grad_norm': 0.16547827422618866, 'learning_rate': 0.00016837554190082208, 'ppl': 1.7461, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4549.83, 'total_tokens': 27743605, 'epoch': 0.79}
-
 26%|█████████████████████████████▎                                                                                 | 659/2499 [1:19:04<3:12:24,  6.27s/it]
 26%|█████████████████████████████▎                                                                                 | 660/2499 [1:19:10<3:12:16,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5474, 'grad_norm': 0.15572473406791687, 'learning_rate': 0.00016828338398288965, 'ppl': 1.7288, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4563.37, 'total_tokens': 27772212, 'epoch': 0.79}
-
 26%|█████████████████████████████▎                                                                                 | 660/2499 [1:19:10<3:12:16,  6.27s/it]
 26%|█████████████████████████████▎                                                                                 | 661/2499 [1:19:17<3:11:56,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5487, 'grad_norm': 0.2158125638961792, 'learning_rate': 0.00016819111728098065, 'ppl': 1.731, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4361.97, 'total_tokens': 27799457, 'epoch': 0.79}
-
 26%|█████████████████████████████▎                                                                                 | 661/2499 [1:19:17<3:11:56,  6.27s/it]
 26%|█████████████████████████████▍                                                                                 | 662/2499 [1:19:23<3:11:52,  6.27s/it]
                                                                                                                                                           
{'loss': 0.534, 'grad_norm': 0.15643706917762756, 'learning_rate': 0.0001680987419420875, 'ppl': 1.7057, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4173.77, 'total_tokens': 27825617, 'epoch': 0.79}
-
 26%|█████████████████████████████▍                                                                                 | 662/2499 [1:19:23<3:11:52,  6.27s/it]
 27%|█████████████████████████████▍                                                                                 | 663/2499 [1:19:29<3:11:57,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5736, 'grad_norm': 0.16257119178771973, 'learning_rate': 0.0001680062581133757, 'ppl': 1.7746, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4626.03, 'total_tokens': 27854685, 'epoch': 0.8}
-
 27%|█████████████████████████████▍                                                                                 | 663/2499 [1:19:29<3:11:57,  6.27s/it]
 27%|█████████████████████████████▍                                                                                 | 664/2499 [1:19:35<3:11:49,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5828, 'grad_norm': 0.1598690301179886, 'learning_rate': 0.0001679136659421835, 'ppl': 1.791, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4542.55, 'total_tokens': 27883155, 'epoch': 0.8}
-
 27%|█████████████████████████████▍                                                                                 | 664/2499 [1:19:35<3:11:49,  6.27s/it]
 27%|█████████████████████████████▌                                                                                 | 665/2499 [1:19:42<3:11:28,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5444, 'grad_norm': 0.15730322897434235, 'learning_rate': 0.0001678209655760219, 'ppl': 1.7236, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4327.13, 'total_tokens': 27910167, 'epoch': 0.8}
-
 27%|█████████████████████████████▌                                                                                 | 665/2499 [1:19:42<3:11:28,  6.26s/it]
 27%|█████████████████████████████▌                                                                                 | 666/2499 [1:19:48<3:11:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5888, 'grad_norm': 0.1599961817264557, 'learning_rate': 0.00016772815716257412, 'ppl': 1.8018, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4567.61, 'total_tokens': 27938805, 'epoch': 0.8}
-
 27%|█████████████████████████████▌                                                                                 | 666/2499 [1:19:48<3:11:26,  6.27s/it]
 27%|█████████████████████████████▋                                                                                 | 667/2499 [1:19:54<3:11:08,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5983, 'grad_norm': 0.16296197474002838, 'learning_rate': 0.0001676352408496956, 'ppl': 1.819, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4387.08, 'total_tokens': 27966196, 'epoch': 0.8}
-
 27%|█████████████████████████████▋                                                                                 | 667/2499 [1:19:54<3:11:08,  6.26s/it]
 27%|█████████████████████████████▋                                                                                 | 668/2499 [1:20:01<3:11:24,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5828, 'grad_norm': 0.16709184646606445, 'learning_rate': 0.00016754221678541367, 'ppl': 1.791, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4387.36, 'total_tokens': 27993823, 'epoch': 0.8}
-
 27%|█████████████████████████████▋                                                                                 | 668/2499 [1:20:01<3:11:24,  6.27s/it]
 27%|█████████████████████████████▋                                                                                 | 669/2499 [1:20:07<3:11:20,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5743, 'grad_norm': 0.1495347023010254, 'learning_rate': 0.00016744908511792726, 'ppl': 1.7759, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4491.57, 'total_tokens': 28021994, 'epoch': 0.8}
-
 27%|█████████████████████████████▋                                                                                 | 669/2499 [1:20:07<3:11:20,  6.27s/it]
 27%|█████████████████████████████▊                                                                                 | 670/2499 [1:20:13<3:11:09,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5361, 'grad_norm': 0.16424506902694702, 'learning_rate': 0.00016735584599560682, 'ppl': 1.7093, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4433.77, 'total_tokens': 28049760, 'epoch': 0.8}
-
 27%|█████████████████████████████▊                                                                                 | 670/2499 [1:20:13<3:11:09,  6.27s/it]
 27%|█████████████████████████████▊                                                                                 | 671/2499 [1:20:19<3:10:58,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5669, 'grad_norm': 0.15702269971370697, 'learning_rate': 0.00016726249956699395, 'ppl': 1.7628, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4422.43, 'total_tokens': 28077446, 'epoch': 0.81}
-
 27%|█████████████████████████████▊                                                                                 | 671/2499 [1:20:19<3:10:58,  6.27s/it]
 27%|█████████████████████████████▊                                                                                 | 672/2499 [1:20:26<3:10:48,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5313, 'grad_norm': 0.14038637280464172, 'learning_rate': 0.00016716904598080111, 'ppl': 1.7011, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4663.47, 'total_tokens': 28106634, 'epoch': 0.81}
-
 27%|█████████████████████████████▊                                                                                 | 672/2499 [1:20:26<3:10:48,  6.27s/it]
 27%|█████████████████████████████▉                                                                                 | 673/2499 [1:20:32<3:10:38,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5654, 'grad_norm': 0.15101970732212067, 'learning_rate': 0.00016707548538591168, 'ppl': 1.7602, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4560.22, 'total_tokens': 28135170, 'epoch': 0.81}
-
 27%|█████████████████████████████▉                                                                                 | 673/2499 [1:20:32<3:10:38,  6.26s/it]
 27%|█████████████████████████████▉                                                                                 | 674/2499 [1:20:38<3:10:42,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5556, 'grad_norm': 0.15208765864372253, 'learning_rate': 0.0001669818179313793, 'ppl': 1.743, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.13, 'tokens_per_second_per_gpu': 4615.18, 'total_tokens': 28164153, 'epoch': 0.81}
-
 27%|█████████████████████████████▉                                                                                 | 674/2499 [1:20:38<3:10:42,  6.27s/it]
 27%|█████████████████████████████▉                                                                                 | 675/2499 [1:20:44<3:10:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5555, 'grad_norm': 0.1590418815612793, 'learning_rate': 0.000166888043766428, 'ppl': 1.7428, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4408.51, 'total_tokens': 28191869, 'epoch': 0.81}
-
 27%|█████████████████████████████▉                                                                                 | 675/2499 [1:20:44<3:10:47,  6.28s/it]
 27%|██████████████████████████████                                                                                 | 676/2499 [1:20:51<3:10:43,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6053, 'grad_norm': 0.1561027318239212, 'learning_rate': 0.0001667941630404517, 'ppl': 1.8318, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4736.35, 'total_tokens': 28221602, 'epoch': 0.81}
-
 27%|██████████████████████████████                                                                                 | 676/2499 [1:20:51<3:10:43,  6.28s/it]
 27%|██████████████████████████████                                                                                 | 677/2499 [1:20:57<3:10:28,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5262, 'grad_norm': 0.15915250778198242, 'learning_rate': 0.00016670017590301423, 'ppl': 1.6925, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4611.16, 'total_tokens': 28250459, 'epoch': 0.81}
-
 27%|██████████████████████████████                                                                                 | 677/2499 [1:20:57<3:10:28,  6.27s/it]
 27%|██████████████████████████████                                                                                 | 678/2499 [1:21:03<3:10:14,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5602, 'grad_norm': 0.14580583572387695, 'learning_rate': 0.0001666060825038488, 'ppl': 1.751, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4565.83, 'total_tokens': 28279025, 'epoch': 0.81}
-
 27%|██████████████████████████████                                                                                 | 678/2499 [1:21:03<3:10:14,  6.27s/it]
 27%|██████████████████████████████▏                                                                                | 679/2499 [1:21:10<3:10:01,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5398, 'grad_norm': 0.14442190527915955, 'learning_rate': 0.00016651188299285802, 'ppl': 1.7157, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4448.55, 'total_tokens': 28306844, 'epoch': 0.82}
-
 27%|██████████████████████████████▏                                                                                | 679/2499 [1:21:10<3:10:01,  6.26s/it]
 27%|██████████████████████████████▏                                                                                | 680/2499 [1:21:16<3:09:49,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5656, 'grad_norm': 0.1592138707637787, 'learning_rate': 0.00016641757752011344, 'ppl': 1.7605, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4508.01, 'total_tokens': 28335030, 'epoch': 0.82}
-
 27%|██████████████████████████████▏                                                                                | 680/2499 [1:21:16<3:09:49,  6.26s/it]
 27%|██████████████████████████████▏                                                                                | 681/2499 [1:21:22<3:09:48,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5587, 'grad_norm': 0.15959708392620087, 'learning_rate': 0.00016632316623585553, 'ppl': 1.7484, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4301.24, 'total_tokens': 28361993, 'epoch': 0.82}
-
 27%|██████████████████████████████▏                                                                                | 681/2499 [1:21:22<3:09:48,  6.26s/it]
 27%|██████████████████████████████▎                                                                                | 682/2499 [1:21:28<3:09:39,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5892, 'grad_norm': 0.15612153708934784, 'learning_rate': 0.0001662286492904933, 'ppl': 1.8025, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4510.28, 'total_tokens': 28390207, 'epoch': 0.82}
-
 27%|██████████████████████████████▎                                                                                | 682/2499 [1:21:28<3:09:39,  6.26s/it]
 27%|██████████████████████████████▎                                                                                | 683/2499 [1:21:35<3:09:24,  6.26s/it]
                                                                                                                                                           
{'loss': 0.554, 'grad_norm': 0.14454488456249237, 'learning_rate': 0.00016613402683460398, 'ppl': 1.7402, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4489.04, 'total_tokens': 28418241, 'epoch': 0.82}
-
 27%|██████████████████████████████▎                                                                                | 683/2499 [1:21:35<3:09:24,  6.26s/it]
 27%|██████████████████████████████▍                                                                                | 684/2499 [1:21:41<3:09:24,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5568, 'grad_norm': 0.16239210963249207, 'learning_rate': 0.00016603929901893305, 'ppl': 1.7451, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4551.45, 'total_tokens': 28446759, 'epoch': 0.82}
-
 27%|██████████████████████████████▍                                                                                | 684/2499 [1:21:41<3:09:24,  6.26s/it]
 27%|██████████████████████████████▍                                                                                | 685/2499 [1:21:47<3:09:15,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5374, 'grad_norm': 0.15430286526679993, 'learning_rate': 0.0001659444659943938, 'ppl': 1.7116, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4503.42, 'total_tokens': 28474926, 'epoch': 0.82}
-
 27%|██████████████████████████████▍                                                                                | 685/2499 [1:21:47<3:09:15,  6.26s/it]
 27%|██████████████████████████████▍                                                                                | 686/2499 [1:21:53<3:09:13,  6.26s/it]
                                                                                                                                                           
{'loss': 0.6042, 'grad_norm': 0.15854589641094208, 'learning_rate': 0.00016584952791206704, 'ppl': 1.8298, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4610.25, 'total_tokens': 28503808, 'epoch': 0.82}
-
 27%|██████████████████████████████▍                                                                                | 686/2499 [1:21:53<3:09:13,  6.26s/it]
 27%|██████████████████████████████▌                                                                                | 687/2499 [1:22:00<3:09:18,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5621, 'grad_norm': 0.1579902619123459, 'learning_rate': 0.0001657544849232011, 'ppl': 1.7544, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4556.28, 'total_tokens': 28532424, 'epoch': 0.82}
-
 27%|██████████████████████████████▌                                                                                | 687/2499 [1:22:00<3:09:18,  6.27s/it]
 28%|██████████████████████████████▌                                                                                | 688/2499 [1:22:06<3:09:14,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5661, 'grad_norm': 0.14277489483356476, 'learning_rate': 0.00016565933717921128, 'ppl': 1.7614, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4680.69, 'total_tokens': 28561775, 'epoch': 0.83}
-
 28%|██████████████████████████████▌                                                                                | 688/2499 [1:22:06<3:09:14,  6.27s/it]
 28%|██████████████████████████████▌                                                                                | 689/2499 [1:22:12<3:08:58,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5377, 'grad_norm': 0.14523279666900635, 'learning_rate': 0.00016556408483167986, 'ppl': 1.7121, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4541.27, 'total_tokens': 28590156, 'epoch': 0.83}
-
 28%|██████████████████████████████▌                                                                                | 689/2499 [1:22:12<3:08:58,  6.26s/it]
 28%|██████████████████████████████▋                                                                                | 690/2499 [1:22:18<3:08:46,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5416, 'grad_norm': 0.16199174523353577, 'learning_rate': 0.00016546872803235578, 'ppl': 1.7188, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4446.12, 'total_tokens': 28617950, 'epoch': 0.83}
-
 28%|██████████████████████████████▋                                                                                | 690/2499 [1:22:18<3:08:46,  6.26s/it]
 28%|██████████████████████████████▋                                                                                | 691/2499 [1:22:25<3:08:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5335, 'grad_norm': 0.15208809077739716, 'learning_rate': 0.0001653732669331543, 'ppl': 1.7049, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4633.04, 'total_tokens': 28647033, 'epoch': 0.83}
-
 28%|██████████████████████████████▋                                                                                | 691/2499 [1:22:25<3:08:50,  6.27s/it]
 28%|██████████████████████████████▋                                                                                | 692/2499 [1:22:31<3:08:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5125, 'grad_norm': 0.15709447860717773, 'learning_rate': 0.00016527770168615698, 'ppl': 1.6695, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4624.44, 'total_tokens': 28676054, 'epoch': 0.83}
-
 28%|██████████████████████████████▋                                                                                | 692/2499 [1:22:31<3:08:50,  6.27s/it]
 28%|██████████████████████████████▊                                                                                | 693/2499 [1:22:37<3:09:02,  6.28s/it]
                                                                                                                                                           
{'loss': 0.4875, 'grad_norm': 0.1479036509990692, 'learning_rate': 0.00016518203244361116, 'ppl': 1.6282, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4542.67, 'total_tokens': 28704686, 'epoch': 0.83}
-
 28%|██████████████████████████████▊                                                                                | 693/2499 [1:22:37<3:09:02,  6.28s/it]
 28%|██████████████████████████████▊                                                                                | 694/2499 [1:22:44<3:08:55,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6043, 'grad_norm': 0.172959104180336, 'learning_rate': 0.00016508625935792996, 'ppl': 1.83, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4336.96, 'total_tokens': 28731899, 'epoch': 0.83}
-
 28%|██████████████████████████████▊                                                                                | 694/2499 [1:22:44<3:08:55,  6.28s/it]
 28%|██████████████████████████████▊                                                                                | 695/2499 [1:22:50<3:08:44,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5326, 'grad_norm': 0.154370978474617, 'learning_rate': 0.0001649903825816918, 'ppl': 1.7034, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4565.64, 'total_tokens': 28760516, 'epoch': 0.83}
-
 28%|██████████████████████████████▊                                                                                | 695/2499 [1:22:50<3:08:44,  6.28s/it]
 28%|██████████████████████████████▉                                                                                | 696/2499 [1:22:56<3:08:24,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5108, 'grad_norm': 0.157211035490036, 'learning_rate': 0.00016489440226764051, 'ppl': 1.6666, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4367.51, 'total_tokens': 28787806, 'epoch': 0.84}
-
 28%|██████████████████████████████▉                                                                                | 696/2499 [1:22:56<3:08:24,  6.27s/it]
 28%|██████████████████████████████▉                                                                                | 697/2499 [1:23:02<3:08:08,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5681, 'grad_norm': 0.1627659946680069, 'learning_rate': 0.0001647983185686847, 'ppl': 1.7649, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4431.29, 'total_tokens': 28815504, 'epoch': 0.84}
-
 28%|██████████████████████████████▉                                                                                | 697/2499 [1:23:02<3:08:08,  6.26s/it]
 28%|███████████████████████████████                                                                                | 698/2499 [1:23:09<3:08:02,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5997, 'grad_norm': 0.1593720018863678, 'learning_rate': 0.00016470213163789765, 'ppl': 1.8216, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4492.52, 'total_tokens': 28843640, 'epoch': 0.84}
-
 28%|███████████████████████████████                                                                                | 698/2499 [1:23:09<3:08:02,  6.26s/it]
 28%|██████████████████████████████���                                                                                | 699/2499 [1:23:15<3:07:55,  6.26s/it]
                                                                                                                                                           
{'loss': 0.6004, 'grad_norm': 0.1652214229106903, 'learning_rate': 0.00016460584162851727, 'ppl': 1.8228, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4455.31, 'total_tokens': 28871533, 'epoch': 0.84}
-
 28%|███████████████████████████████                                                                                | 699/2499 [1:23:15<3:07:55,  6.26s/it]
 28%|███████████████████████████████                                                                                | 700/2499 [1:23:21<3:08:09,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6045, 'grad_norm': 0.4151879847049713, 'learning_rate': 0.00016450944869394554, 'ppl': 1.8303, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.19, 'tokens_per_second_per_gpu': 4499.12, 'total_tokens': 28899871, 'epoch': 0.84}
-
 28%|███████████████████████████████                                                                                | 700/2499 [1:23:21<3:08:09,  6.28s/it][2025-12-28 12:28:57,781] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 12:29:00,510] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 1.3375587463378906
-[2025-12-28 12:29:01,570] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 1.0599887371063232
-[2025-12-28 12:29:02,589] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 1.0184319019317627
-[2025-12-28 12:29:03,500] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.9104523658752441
-[2025-12-28 12:29:03,500] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
+
  0%|                                                                                                                               | 0/72 [00:00<?, ?it/s]
+
  3%|███▎                                                                                                                   | 2/72 [00:02<01:19,  1.13s/it]
+
  4%|████▉                                                                                                                  | 3/72 [00:04<01:40,  1.46s/it]
+
  6%|██████▌                                                                                                                | 4/72 [00:06<01:50,  1.62s/it]
+
  7%|████████▎                                                                                                              | 5/72 [00:07<01:54,  1.71s/it]
+
  8%|█████████▉                                                                                                             | 6/72 [00:09<01:56,  1.76s/it]
+
 10%|███████████▌                                                                                                           | 7/72 [00:11<01:58,  1.82s/it]
+
 11%|█████████████▏                                                                                                         | 8/72 [00:13<01:57,  1.84s/it]
+
 12%|██████████████▉                                                                                                        | 9/72 [00:15<01:57,  1.86s/it]
+
 14%|████████████████▍                                                                                                     | 10/72 [00:17<01:56,  1.87s/it]
+
 15%|██████████████████                                                                                                    | 11/72 [00:19<01:54,  1.88s/it]
+
 17%|███████████████████▋                                                                                                  | 12/72 [00:21<01:53,  1.89s/it]
+
 18%|█████████████████████▎                                                                                                | 13/72 [00:23<01:51,  1.89s/it]
+
 19%|██████████████████████▉                                                                                               | 14/72 [00:25<01:50,  1.90s/it]
+
 21%|████████████████████████▌                                                                                             | 15/72 [00:26<01:48,  1.90s/it]
+
 22%|██████████████████████████▏                                                                                           | 16/72 [00:28<01:45,  1.89s/it]
+
 24%|███████████████████████████▊                                                                                          | 17/72 [00:30<01:43,  1.88s/it]
+
 25%|█████████████████████████████▌                                                                                        | 18/72 [00:32<01:42,  1.89s/it]
+
 26%|███████████████████████████████▏                                                                                      | 19/72 [00:34<01:40,  1.89s/it]
+
 28%|████████████████████████████████▊                                                                                     | 20/72 [00:36<01:38,  1.90s/it]
+
 29%|██████████████████████████████████▍                                                                                   | 21/72 [00:38<01:36,  1.90s/it]
+
 31%|████████████████████████████████████                                                                                  | 22/72 [00:40<01:35,  1.90s/it]
+
 32%|█████████████████████████████████████▋                                                                                | 23/72 [00:42<01:33,  1.90s/it]
+
 33%|███████████████████████████████████████▎                                                                              | 24/72 [00:44<01:31,  1.90s/it]
+
 35%|████████████████████████████████████████▉                                                                             | 25/72 [00:45<01:29,  1.91s/it]
+
 36%|██████████████████████████████████████████▌                                                                           | 26/72 [00:47<01:27,  1.90s/it]
+
 38%|████████████████████████████████████████████▎                                                                         | 27/72 [00:49<01:25,  1.89s/it]
+
 39%|█████████████████████████████████████████████▉                                                                        | 28/72 [00:51<01:23,  1.89s/it]
+
 40%|███████████████████████████████████████████████▌                                                                      | 29/72 [00:53<01:21,  1.89s/it]
+
 42%|█████████████████████████████████████████████████▏                                                                    | 30/72 [00:55<01:20,  1.91s/it]
+
 43%|██████████████████████████████████████████████████▊                                                                   | 31/72 [00:57<01:18,  1.91s/it]
+
 44%|████████████████████████████████████████████████████▍                                                                 | 32/72 [00:59<01:17,  1.94s/it]
+
 46%|██████████████████████████████████████████████████████                                                                | 33/72 [01:01<01:15,  1.93s/it]
+
 47%|███████████████████████████████████████████████████████▋                                                              | 34/72 [01:03<01:13,  1.93s/it]
+
 49%|█████████████████████████████████████████████████████████▎                                                            | 35/72 [01:05<01:11,  1.92s/it]
+
 50%|███████████████���███████████████████████████████████████████                                                           | 36/72 [01:07<01:08,  1.91s/it]
+
 51%|████████████████████████████████████████████████████████████▋                                                         | 37/72 [01:08<01:06,  1.90s/it]
+
 53%|██████████████████████████████████████████████████████████████▎                                                       | 38/72 [01:10<01:04,  1.89s/it]
+
 54%|███████████████████████████████████████████████████████████████▉                                                      | 39/72 [01:12<01:02,  1.91s/it]
+
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 40/72 [01:15<01:06,  2.07s/it]
+
 57%|███████████████████████████████████████████████████████████████████▏                                                  | 41/72 [01:17<01:02,  2.02s/it]
+
 58%|████████████████████████████████████████████████████████████████████▊                                                 | 42/72 [01:18<00:59,  1.99s/it]
+
 60%|██████████████████████████████████████████████████████████████████████▍                                               | 43/72 [01:20<00:56,  1.96s/it]
+
 61%|████████████████████████████████████████████████████████████████████████                                              | 44/72 [01:22<00:54,  1.94s/it]
+
 62%|█████████████████████████████████████████████████████████████████████████▊                                            | 45/72 [01:24<00:52,  1.93s/it]
+
 64%|███████████████████████████████████████████████████████████████████████████▍                                          | 46/72 [01:26<00:50,  1.92s/it]
+
 65%|█████████████████████████████████████████████████████████████████████████████                                         | 47/72 [01:28<00:47,  1.91s/it]
+
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 48/72 [01:30<00:45,  1.90s/it]
+
 68%|████████████████████████████████████████████████████████████████████████████████▎                                     | 49/72 [01:32<00:44,  1.93s/it]
+
 69%|█████████████████████████████████████████████████████████████████████████████████▉                                    | 50/72 [01:34<00:42,  1.92s/it]
+
 71%|███████████████████████████████████████████████████████████████████████████████████▌                                  | 51/72 [01:36<00:40,  1.91s/it]
+
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 52/72 [01:38<00:38,  1.91s/it]
+
 74%|██████████████████████████████████████████████████████████████████████████████████████▊                               | 53/72 [01:39<00:36,  1.91s/it]
+
 75%|████████████████████████████████████████████████████████████████████████████████████████▌                             | 54/72 [01:41<00:34,  1.91s/it]
+
 76%|██████████████████████████████████████████████████████████████████████████████████████████▏                           | 55/72 [01:43<00:32,  1.91s/it]
+
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 56/72 [01:45<00:30,  1.91s/it]
+
 79%|█████████████████████████████████████████████████████████████████████████████████████████████▍                        | 57/72 [01:47<00:28,  1.90s/it]
+
 81%|███████████████████████████████████████████████████████████████████████████████████████████████                       | 58/72 [01:49<00:26,  1.89s/it]
+
 82%|████████████████████████████████████████████████████████████████████████████████████████████████▋                     | 59/72 [01:51<00:24,  1.89s/it]
+
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 60/72 [01:53<00:22,  1.89s/it]
+
 85%|███████████████████████████████████████████████████████████████████████████████████████████████████▉                  | 61/72 [01:55<00:20,  1.89s/it]
+
 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌                | 62/72 [01:56<00:18,  1.90s/it]
+
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎              | 63/72 [01:58<00:17,  1.90s/it]
+
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 64/72 [02:00<00:15,  1.90s/it]
+
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 65/72 [02:02<00:13,  1.90s/it]
+
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 66/72 [02:04<00:11,  1.90s/it]
+
 93%|█████████████████████████████��███████████████████████████████████████████████████████████████████████████████▊        | 67/72 [02:06<00:09,  1.91s/it]
+
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 68/72 [02:08<00:07,  1.90s/it]
+
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████     | 69/72 [02:10<00:05,  1.90s/it]
+
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 70/72 [02:12<00:03,  1.97s/it]
+
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 71/72 [02:14<00:01,  1.97s/it]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:16<00:00,  1.99s/it]
                                                                                                                                                           
+
                                                                                                                                                           
{'eval_loss': 0.540988564491272, 'eval_runtime': 138.0264, 'eval_samples_per_second': 5.289, 'eval_steps_per_second': 1.058, 'eval_ppl': 1.7177, 'memory/max_active (GiB)': 19.1, 'memory/max_allocated (GiB)': 19.1, 'memory/device_reserved (GiB)': 139.02, 'epoch': 0.75}
+
 25%|███████████████████████████▌                                                                                  | 1000/3996 [1:16:53<3:40:01,  4.41s/it]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:16<00:00,  1.99s/it]
+
                                                                                                                                                           [2025-12-29 04:06:38,383] [INFO] [axolotl.core.trainers.base._save:692] [PID:3751] Saving model checkpoint to ./outputs/luau-codellama-h200-fast/checkpoint-1000
+
 25%|███████████████████████████▎                                                                                 | 1001/3996 [1:16:57<38:35:51, 46.39s/it]
 25%|███████████████████████████▎                                                                                 | 1002/3996 [1:17:01<28:03:49, 33.74s/it]
 25%|███████████████████████████▎                                                                                 | 1003/3996 [1:17:06<20:49:21, 25.05s/it]
 25%|███████████████████████████▍                                                                                 | 1004/3996 [1:17:10<15:37:18, 18.80s/it]
 25%|███████████████████████████▍                                                                                 | 1005/3996 [1:17:14<11:58:44, 14.42s/it]
 25%|███████████████████████████▋                                                                                  | 1006/3996 [1:17:19<9:27:10, 11.38s/it]
 25%|███████████████████████████���                                                                                  | 1007/3996 [1:17:23<7:39:59,  9.23s/it]
 25%|███████████████████████████▋                                                                                  | 1008/3996 [1:17:27<6:24:38,  7.72s/it]
 25%|███████████████████████████▊                                                                                  | 1009/3996 [1:17:31<5:31:55,  6.67s/it]
 25%|███████████████████████████▊                                                                                  | 1010/3996 [1:17:36<5:03:38,  6.10s/it]
 25%|███████████████████████████▊                                                                                  | 1011/3996 [1:17:40<4:35:23,  5.54s/it]
 25%|███████████████████████████▊                                                                                  | 1012/3996 [1:17:44<4:15:56,  5.15s/it]
 25%|███████████████████████████▉                                                                                  | 1013/3996 [1:17:49<4:01:47,  4.86s/it]
 25%|███████████████████████████▉                                                                                  | 1014/3996 [1:17:53<3:51:57,  4.67s/it]
 25%|███████████████████████████▉                                                                                  | 1015/3996 [1:17:57<3:44:59,  4.53s/it]
 25%|███████████████████████████▉                                                                                  | 1016/3996 [1:18:01<3:40:04,  4.43s/it]
 25%|███████████████████████████▉                                                                                  | 1017/3996 [1:18:06<3:44:43,  4.53s/it]
 25%|████████████████████████████                                                                                  | 1018/3996 [1:18:10<3:40:14,  4.44s/it]
 26%|████████████████████████████                                                                                  | 1019/3996 [1:18:14<3:36:42,  4.37s/it]
 26%|████████████████████████████                                                                                  | 1020/3996 [1:18:19<3:34:03,  4.32s/it]
 26%|████████████████████████████                                                                                  | 1021/3996 [1:18:23<3:32:21,  4.28s/it]
 26%|████████████████████████████▏                                                                                 | 1022/3996 [1:18:27<3:35:31,  4.35s/it]
 26%|████████████████████████████▏                                                                                 | 1023/3996 [1:18:31<3:33:12,  4.30s/it]
 26%|████████████████████████████▏                                                                                 | 1024/3996 [1:18:36<3:39:51,  4.44s/it]
 26%|████████████████████████████▏                                                                                 | 1025/3996 [1:18:40<3:36:16,  4.37s/it]
                                                                                                                                                           
{'loss': 0.549, 'grad_norm': 0.2199818342924118, 'learning_rate': 0.00017350336741329413, 'ppl': 1.7315, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4129.73, 'total_tokens': 20870820, 'epoch': 0.77}
+
 26%|████████████████████████████▏                                                                                 | 1025/3996 [1:18:40<3:36:16,  4.37s/it]
 26%|████████████████████████████▏                                                                                 | 1026/3996 [1:18:45<3:33:50,  4.32s/it]
 26%|████████████████████████████▎                                                                                 | 1027/3996 [1:18:49<3:31:49,  4.28s/it]
 26%|███████████████████████��████▎                                                                                 | 1028/3996 [1:18:53<3:30:46,  4.26s/it]
 26%|████████████████████████████▎                                                                                 | 1029/3996 [1:18:57<3:29:55,  4.25s/it]
 26%|████████████████████████████▎                                                                                 | 1030/3996 [1:19:01<3:29:29,  4.24s/it]
 26%|████████████████████████████▍                                                                                 | 1031/3996 [1:19:06<3:36:59,  4.39s/it]
 26%|████████████████████████████▍                                                                                 | 1032/3996 [1:19:10<3:34:25,  4.34s/it]
 26%|████████████████████████████▍                                                                                 | 1033/3996 [1:19:15<3:32:38,  4.31s/it]
 26%|████████████████████████████▍                                                                                 | 1034/3996 [1:19:19<3:31:06,  4.28s/it]
 26%|████████████████████████████▍                                                                                 | 1035/3996 [1:19:23<3:30:24,  4.26s/it]
 26%|████████████████████████████▌                                                                                 | 1036/3996 [1:19:27<3:29:45,  4.25s/it]
 26%|████████████████████████████▌                                                                                 | 1037/3996 [1:19:32<3:29:14,  4.24s/it]
 26%|████████████████████████████▌                                                                                 | 1038/3996 [1:19:36<3:36:45,  4.40s/it]
 26%|████████████████████████████▌                                                                                 | 1039/3996 [1:19:41<3:34:05,  4.34s/it]
 26%|████████████████████████████▋                                                                                 | 1040/3996 [1:19:45<3:35:13,  4.37s/it]
 26%|████████████████████████████▋                                                                                 | 1041/3996 [1:19:49<3:32:42,  4.32s/it]
 26%|████████████████████████████▋                                                                                 | 1042/3996 [1:19:53<3:31:23,  4.29s/it]
 26%|████████████████████████████▋                                                                                 | 1043/3996 [1:19:58<3:30:10,  4.27s/it]
 26%|████████████████████████████▋                                                                                 | 1044/3996 [1:20:02<3:29:38,  4.26s/it]
 26%|████████████████████████████▊                                                                                 | 1045/3996 [1:20:07<3:36:54,  4.41s/it]
 26%|████████████████████████████▊                                                                                 | 1046/3996 [1:20:11<3:34:08,  4.36s/it]
 26%|████████████████████████████▊                                                                                 | 1047/3996 [1:20:15<3:32:14,  4.32s/it]
 26%|████████████████████████████▊                                                                                 | 1048/3996 [1:20:19<3:30:34,  4.29s/it]
 26%|████████████████████████████▉                                                                                 | 1049/3996 [1:20:24<3:29:45,  4.27s/it]
 26%|████████████████████████████▉                                                                                 | 1050/3996 [1:20:28<3:28:52,  4.25s/it]
                                                                                                                                                           
{'loss': 0.5615, 'grad_norm': 0.19783177971839905, 'learning_rate': 0.0001721216769951596, 'ppl': 1.7533, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4243.63, 'total_tokens': 21317982, 'epoch': 0.79}
+
 26%|████████████████████████████▉                                                                                 | 1050/3996 [1:20:28<3:28:52,  4.25s/it]
 26%|████████████████████████████▉                                                                                 | 1051/3996 [1:20:32<3:28:09,  4.24s/it]
 26%|████████████████████████████▉                                                                                 | 1052/3996 [1:20:37<3:36:40,  4.42s/it]
 26%|████████████████████████████▉                                                                                 | 1053/3996 [1:20:41<3:33:34,  4.35s/it]
 26%|█████████████████████████████                                                                                 | 1054/3996 [1:20:45<3:31:31,  4.31s/it]
 26%|█████████████████████████████                                                                                 | 1055/3996 [1:20:49<3:29:49,  4.28s/it]
 26%|█████████████████████████████                                                                                 | 1056/3996 [1:20:54<3:29:07,  4.27s/it]
 26%|█████████████████████████████                                                                                 | 1057/3996 [1:20:58<3:28:02,  4.25s/it]
 26%|█████████████████████████████                                                                                 | 1058/3996 [1:21:02<3:27:52,  4.25s/it]
 27%|█████████████████████████████▏                                                                                | 1059/3996 [1:21:07<3:34:51,  4.39s/it]
 27%|█████████████████████████████▏                                                                                | 1060/3996 [1:21:11<3:32:27,  4.34s/it]
 27%|█████████████████████████████▏                                                                                | 1061/3996 [1:21:15<3:30:24,  4.30s/it]
 27%|█████████████████████████████▏                                                                                | 1062/3996 [1:21:19<3:28:49,  4.27s/it]
 27%|█████████████████████████████▎                                                                                | 1063/3996 [1:21:24<3:28:02,  4.26s/it]
 27%|█████████████████████████████▎                                                                                | 1064/3996 [1:21:28<3:27:05,  4.24s/it]
 27%|█████████████████████████████▎                                                                                | 1065/3996 [1:21:32<3:26:44,  4.23s/it]
 27%|█████████████████████████████▎                                                                                | 1066/3996 [1:21:37<3:35:37,  4.42s/it]
 27%|█████████████████████████████▎                                                                                | 1067/3996 [1:21:41<3:32:38,  4.36s/it]
 27%|█████████████████████████████▍                                                                                | 1068/3996 [1:21:45<3:30:16,  4.31s/it]
 27%|█████████████████████████████▍                                                                                | 1069/3996 [1:21:50<3:28:51,  4.28s/it]
 27%|█████████████████████████████▍                                                                                | 1070/3996 [1:21:54<3:27:56,  4.26s/it]
 27%|█████████████████████████████▍                                                                                | 1071/3996 [1:21:58<3:26:49,  4.24s/it]
 27%|█████████████████████████████▌                                                                                | 1072/3996 [1:22:02<3:26:26,  4.24s/it]
 27%|█████████████████████████████▌                                                                                | 1073/3996 [1:22:07<3:33:41,  4.39s/it]
 27%|█████████████████████████████▌                                                                                | 1074/3996 [1:22:11<3:31:11,  4.34s/it]
 27%|█████████████████████████████▌                                                                                | 1075/3996 [1:22:15<3:30:03,  4.31s/it]
                                                                                                                                                           
{'loss': 0.5557, 'grad_norm': 0.1678430140018463, 'learning_rate': 0.00017071067811865476, 'ppl': 1.7432, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4092.04, 'total_tokens': 21754087, 'epoch': 0.81}
+
 27%|█████████████████████████████▌                                                                                | 1075/3996 [1:22:15<3:30:03,  4.31s/it]
 27%|█████████████████████████████▌                                                                                | 1076/3996 [1:22:21<3:42:22,  4.57s/it]
 27%|█████████████████████████████▋                                                                                | 1077/3996 [1:22:25<3:37:09,  4.46s/it]
 27%|█████████████████████████████▋                                                                                | 1078/3996 [1:22:29<3:33:13,  4.38s/it]
 27%|█████████████████████████████▋                                                                                | 1079/3996 [1:22:33<3:30:35,  4.33s/it]
 27%|█████████████████████████████▋                                                                                | 1080/3996 [1:22:38<3:36:23,  4.45s/it]
 27%|█████████████████████████████▊                                                                                | 1081/3996 [1:22:42<3:33:04,  4.39s/it]
 27%|█████████████████████████████▊                                                                                | 1082/3996 [1:22:46<3:30:23,  4.33s/it]
 27%|█████████████████████████████▊                                                                                | 1083/3996 [1:22:51<3:28:31,  4.29s/it]
 27%|█████████████████████████████▊                                                                                | 1084/3996 [1:22:55<3:27:16,  4.27s/it]
 27%|█████████████████████████████▊                                                                                | 1085/3996 [1:22:59<3:26:15,  4.25s/it]
 27%|█████████████████████████████▉                                                                                | 1086/3996 [1:23:03<3:25:29,  4.24s/it]
 27%|█████████████████████████████▉                                                                                | 1087/3996 [1:23:08<3:32:34,  4.38s/it]
 27%|█████████████████████████████▉                                                                                | 1088/3996 [1:23:12<3:30:08,  4.34s/it]
 27%|█████████████████████████████▉                                                                                | 1089/3996 [1:23:16<3:28:03,  4.29s/it]
 27%|██████████████████████████████                                                                                | 1090/3996 [1:23:21<3:32:28,  4.39s/it]
 27%|██████████████████████████████                                                                                | 1091/3996 [1:23:25<3:29:59,  4.34s/it]
 27%|██████████████████████████████                                                                                | 1092/3996 [1:23:29<3:28:00,  4.30s/it]
 27%|██████████████████████████████                                                                                | 1093/3996 [1:23:34<3:26:41,  4.27s/it]
 27%|██████████████████████████████                                                                                | 1094/3996 [1:23:39<3:38:31,  4.52s/it]
 27%|██████████████████████████████▏                                                                               | 1095/3996 [1:23:43<3:34:02,  4.43s/it]
 27%|██████████████████████████████▏                                                                               | 1096/3996 [1:23:47<3:30:51,  4.36s/it]
 27%|██████████████████████████████▏                                                                               | 1097/3996 [1:23:51<3:28:33,  4.32s/it]
 27%|██████████████████████████████▏                                                                               | 1098/3996 [1:23:56<3:27:10,  4.29s/it]
 28%|██████████████████████████████▎                                                                               | 1099/3996 [1:24:00<3:25:59,  4.27s/it]
 28%|██████████████████████████████▎                                                                               | 1100/3996 [1:24:04<3:25:40,  4.26s/it]
                                                                                                                                                           
{'loss': 0.556, 'grad_norm': 0.16523879766464233, 'learning_rate': 0.00016927094417868048, 'ppl': 1.7437, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4187.02, 'total_tokens': 22198779, 'epoch': 0.83}
+
 28%|██████████████████████████████▎                                                                               | 1100/3996 [1:24:04<3:25:40,  4.26s/it]
 28%|██████████████████████████████▎                                                                               | 1101/3996 [1:24:09<3:32:40,  4.41s/it]
 28%|██████████████████████████████▎                                                                               | 1102/3996 [1:24:13<3:29:52,  4.35s/it]
 28%|██████████████████████████████▎                                                                               | 1103/3996 [1:24:17<3:28:31,  4.32s/it]
 28%|██████████████████████████████▍                                                                               | 1104/3996 [1:24:21<3:26:38,  4.29s/it]
 28%|██████████████████████████████▍                                                                               | 1105/3996 [1:24:26<3:25:37,  4.27s/it]
 28%|██████████████████████████████▍                                                                               | 1106/3996 [1:24:30<3:24:48,  4.25s/it]
 28%|██████████████████████████████▍                                                                               | 1107/3996 [1:24:34<3:24:31,  4.25s/it]
 28%|██████████████████████████████▌                                                                               | 1108/3996 [1:24:39<3:31:57,  4.40s/it]
 28%|██████████████████████████████▌                                                                               | 1109/3996 [1:24:43<3:29:23,  4.35s/it]
 28%|██████████████████████████████▌                                                                               | 1110/3996 [1:24:47<3:27:09,  4.31s/it]
 28%|██████████████████████████████▌                                                                               | 1111/3996 [1:24:52<3:25:30,  4.27s/it]
 28%|██████████████████████████████▌                                                                               | 1112/3996 [1:24:56<3:27:52,  4.32s/it]
 28%|██████████████████████████████▋                                                                               | 1113/3996 [1:25:00<3:26:14,  4.29s/it]
 28%|██████████████████████████████▋                                                                               | 1114/3996 [1:25:04<3:24:47,  4.26s/it]
 28%|██████████████████████████████▋                                                                               | 1115/3996 [1:25:09<3:31:34,  4.41s/it]
 28%|██████████████████████████████▋                                                                               | 1116/3996 [1:25:13<3:28:52,  4.35s/it]
 28%|██████████████████████████████▋                                                                               | 1117/3996 [1:25:18<3:26:46,  4.31s/it]
 28%|██████████████████████████████▊                                                                               | 1118/3996 [1:25:22<3:25:07,  4.28s/it]
 28%|██████████████████████████████▊                                                                               | 1119/3996 [1:25:26<3:23:57,  4.25s/it]
 28%|██████████████████████████████▊                                                                               | 1120/3996 [1:25:30<3:23:18,  4.24s/it]
 28%|██████████████████████████████▊                                                                               | 1121/3996 [1:25:34<3:22:55,  4.23s/it]
 28%|██████████████████████████████▉                                                                               | 1122/3996 [1:25:39<3:30:26,  4.39s/it]
 28%|██████████████████████████████▉                                                                               | 1123/3996 [1:25:43<3:27:50,  4.34s/it]
 28%|██████████████████████████████▉                                                                               | 1124/3996 [1:25:48<3:26:00,  4.30s/it]
 28%|██████████████████████████████▉                                                                               | 1125/3996 [1:25:52<3:24:28,  4.27s/it]
                                                                                                                                                           
{'loss': 0.5468, 'grad_norm': 0.18177717924118042, 'learning_rate': 0.00016780306024735382, 'ppl': 1.7277, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4198.97, 'total_tokens': 22639769, 'epoch': 0.84}
+
 28%|██████████████████████████████▉                                                                               | 1125/3996 [1:25:52<3:24:28,  4.27s/it]
 28%|██████████████████████████████▉                                                                               | 1126/3996 [1:25:56<3:23:31,  4.25s/it]
 28%|███████████████████████████████                                                                               | 1127/3996 [1:26:00<3:22:43,  4.24s/it]
 28%|███████████████████████████████                                                                               | 1128/3996 [1:26:04<3:22:13,  4.23s/it]
 28%|███████████████████████████████                                                                               | 1129/3996 [1:26:10<3:36:51,  4.54s/it]
 28%|███████████████████████████████                                                                               | 1130/3996 [1:26:14<3:32:09,  4.44s/it]
 28%|███████████████████████████████▏                                                                              | 1131/3996 [1:26:18<3:28:42,  4.37s/it]
 28%|███████████████████████████████▏                                                                              | 1132/3996 [1:26:22<3:26:12,  4.32s/it]
 28%|███████████████████████████████▏                                                                              | 1133/3996 [1:26:26<3:24:40,  4.29s/it]
 28%|███████████████████████���███████▏                                                                              | 1134/3996 [1:26:31<3:23:26,  4.27s/it]
 28%|███████████████████████████████▏                                                                              | 1135/3996 [1:26:36<3:45:13,  4.72s/it]
 28%|███████████████████████████████▎                                                                              | 1136/3996 [1:26:41<3:45:37,  4.73s/it]
 28%|███████████████████████████████▎                                                                              | 1137/3996 [1:26:45<3:38:03,  4.58s/it]
 28%|███████████████████████████████▎                                                                              | 1138/3996 [1:26:50<3:32:27,  4.46s/it]
 29%|███████████████████████████████▎                                                                              | 1139/3996 [1:26:54<3:28:42,  4.38s/it]
 29%|███████████████████████████████▍                                                                              | 1140/3996 [1:26:58<3:26:17,  4.33s/it]
 29%|███████████████████████████████▍                                                                              | 1141/3996 [1:27:02<3:24:35,  4.30s/it]
 29%|███████████████████████████████▍                                                                              | 1142/3996 [1:27:07<3:23:17,  4.27s/it]
 29%|███████████████████████████████▍                                                                              | 1143/3996 [1:27:11<3:29:56,  4.42s/it]
 29%|███████████████████████████████▍                                                                              | 1144/3996 [1:27:16<3:27:36,  4.37s/it]
 29%|███████████████████████████████▌                                                                              | 1145/3996 [1:27:20<3:25:10,  4.32s/it]
 29%|███████████████████████████████▌                                                                              | 1146/3996 [1:27:24<3:23:19,  4.28s/it]
 29%|███████████████████████████████▌                                                                              | 1147/3996 [1:27:28<3:22:15,  4.26s/it]
 29%|███████████████████████████████▌                                                                              | 1148/3996 [1:27:32<3:21:17,  4.24s/it]
 29%|███████████████████████████████▋                                                                              | 1149/3996 [1:27:36<3:20:26,  4.22s/it]
 29%|███████████████████████████████▋                                                                              | 1150/3996 [1:27:41<3:27:53,  4.38s/it]
                                                                                                                                                           
{'loss': 0.554, 'grad_norm': 0.17299720644950867, 'learning_rate': 0.0001663076228362492, 'ppl': 1.7402, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3762.13, 'total_tokens': 23086742, 'epoch': 0.86}
+
 29%|███████████████████████████████▋                                                                              | 1150/3996 [1:27:41<3:27:53,  4.38s/it]
 29%|███████████████████████████████▋                                                                              | 1151/3996 [1:27:45<3:25:21,  4.33s/it]
 29%|███████████████████████████████▋                                                                              | 1152/3996 [1:27:50<3:23:19,  4.29s/it]
 29%|███████████████████████████████▋                                                                              | 1153/3996 [1:27:54<3:21:54,  4.26s/it]
 29%|███████████████████████████████▊                                                                              | 1154/3996 [1:27:58<3:21:15,  4.25s/it]
 29%|███████████████████████████████▊                                                                              | 1155/3996 [1:28:02<3:20:32,  4.24s/it]
 29%|███████████████████████████████▊                                                                              | 1156/3996 [1:28:06<3:19:56,  4.22s/it]
 29%|███████████████████████████████▊                                                                              | 1157/3996 [1:28:11<3:27:09,  4.38s/it]
 29%|███████████████████████████████▉                                                                              | 1158/3996 [1:28:15<3:24:37,  4.33s/it]
 29%|███████████████████████████████▉                                                                              | 1159/3996 [1:28:20<3:22:45,  4.29s/it]
 29%|███████████████████████████████▉                                                                              | 1160/3996 [1:28:24<3:21:10,  4.26s/it]
 29%|███████████████████████████████▉                                                                              | 1161/3996 [1:28:28<3:20:28,  4.24s/it]
 29%|███████████████████████████████▉                                                                              | 1162/3996 [1:28:32<3:19:48,  4.23s/it]
 29%|████████████████████████████████                                                                              | 1163/3996 [1:28:36<3:19:19,  4.22s/it]
 29%|████████████████████████████████                                                                              | 1164/3996 [1:28:41<3:26:54,  4.38s/it]
 29%|████████████████████████████████                                                                              | 1165/3996 [1:28:45<3:24:16,  4.33s/it]
 29%|████████████████████████████████                                                                              | 1166/3996 [1:28:50<3:22:02,  4.28s/it]
 29%|████████████████████████████████                                                                              | 1167/3996 [1:28:54<3:20:37,  4.25s/it]
 29%|████████████████████████████████▏                                                                             | 1168/3996 [1:28:58<3:19:48,  4.24s/it]
 29%|████████████████████████████████▏                                                                             | 1169/3996 [1:29:02<3:19:14,  4.23s/it]
 29%|████████████████████████████████▏                                                                             | 1170/3996 [1:29:06<3:18:40,  4.22s/it]
 29%|████████████████████████████████▏                                                                             | 1171/3996 [1:29:11<3:25:55,  4.37s/it]
 29%|████████████████████████████████▎                                                                             | 1172/3996 [1:29:15<3:23:39,  4.33s/it]
 29%|████████████████████████████████▎                                                                             | 1173/3996 [1:29:19<3:21:52,  4.29s/it]
 29%|████████████████████████████████▎                                                                             | 1174/3996 [1:29:24<3:20:27,  4.26s/it]
 29%|████████████████████████████████▎                                                                             | 1175/3996 [1:29:28<3:19:29,  4.24s/it]
                                                                                                                                                           
{'loss': 0.5434, 'grad_norm': 0.19112971425056458, 'learning_rate': 0.00016478523965399085, 'ppl': 1.7219, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4205.37, 'total_tokens': 23528106, 'epoch': 0.88}
+
 29%|████████████████████████████████▎                                                                             | 1175/3996 [1:29:28<3:19:29,  4.24s/it]
 29%|████████████████████████████████▎                                                                             | 1176/3996 [1:29:32<3:18:51,  4.23s/it]
 29%|████████████████████████████████▍                                                                             | 1177/3996 [1:29:36<3:18:28,  4.22s/it]
 29%|████████████████████████████████▍                                                                             | 1178/3996 [1:29:41<3:25:38,  4.38s/it]
 30%|████████████████████████████████▍                                                                             | 1179/3996 [1:29:45<3:23:01,  4.32s/it]
 30%|████████████████████████████████▍                                                                             | 1180/3996 [1:29:49<3:20:56,  4.28s/it]
 30%|████████████████████████████████▌                                                                             | 1181/3996 [1:29:54<3:19:36,  4.25s/it]
 30%|████████████████████████████████▌                                                                             | 1182/3996 [1:29:58<3:18:49,  4.24s/it]
 30%|████████████████████████████████▌                                                                             | 1183/3996 [1:30:02<3:18:22,  4.23s/it]
 30%|████████████████████████████████▌                                                                             | 1184/3996 [1:30:06<3:17:55,  4.22s/it]
 30%|████████████████████████████████▌                                                                             | 1185/3996 [1:30:11<3:32:19,  4.53s/it]
 30%|████████████████████████████████▋                                                                             | 1186/3996 [1:30:16<3:27:39,  4.43s/it]
 30%|████████████████████████████████▋                                                                             | 1187/3996 [1:30:20<3:24:13,  4.36s/it]
 30%|████████████████████████████████▋                                                                             | 1188/3996 [1:30:24<3:21:53,  4.31s/it]
 30%|████████████████████████████████▋                                                                             | 1189/3996 [1:30:28<3:20:19,  4.28s/it]
 30%|████████████████████████████████▊                                                                             | 1190/3996 [1:30:32<3:19:08,  4.26s/it]
 30%|████████████████████████████████▊                                                                             | 1191/3996 [1:30:37<3:18:18,  4.24s/it]
 30%|████████████████████████████████▊                                                                             | 1192/3996 [1:30:41<3:25:23,  4.39s/it]
 30%|████████████████████████████████▊                                                                             | 1193/3996 [1:30:46<3:22:49,  4.34s/it]
 30%|████████████████████████████████▊                                                                             | 1194/3996 [1:30:50<3:20:47,  4.30s/it]
 30%|████████████████████████████████▉                                                                             | 1195/3996 [1:30:54<3:19:22,  4.27s/it]
 30%|████████████████████████████████▉                                                                             | 1196/3996 [1:30:58<3:18:29,  4.25s/it]
 30%|████████████████████████████████▉                                                                             | 1197/3996 [1:31:02<3:17:41,  4.24s/it]
 30%|████████████████████████████████▉                                                                             | 1198/3996 [1:31:07<3:17:05,  4.23s/it]
 30%|█████████████████████████████████                                                                             | 1199/3996 [1:31:11<3:24:12,  4.38s/it]
 30%|█████████████████████████████████                                                                             | 1200/3996 [1:31:16<3:21:55,  4.33s/it]
                                                                                                                                                           
{'loss': 0.5362, 'grad_norm': 0.17930163443088531, 'learning_rate': 0.00016323652935929536, 'ppl': 1.7095, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4228.83, 'total_tokens': 23974427, 'epoch': 0.9}
+
 30%|█████████████████████████████████                                                                             | 1200/3996 [1:31:16<3:21:55,  4.33s/it]
 30%|█████████████████████████████████                                                                             | 1201/3996 [1:31:20<3:19:54,  4.29s/it]
 30%|█████████████████████████████████                                                                             | 1202/3996 [1:31:24<3:18:38,  4.27s/it]
 30%|█████████████████████████████████                                                                             | 1203/3996 [1:31:28<3:17:55,  4.25s/it]
 30%|█████████████████████████████████▏                                                                            | 1204/3996 [1:31:32<3:17:07,  4.24s/it]
 30%|█████████████████████████████████▏                                                                            | 1205/3996 [1:31:37<3:16:32,  4.23s/it]
 30%|█████████████████████████████████▏                                                                            | 1206/3996 [1:31:41<3:23:48,  4.38s/it]
 30%|█████████████████████████████████▏                                                                            | 1207/3996 [1:31:46<3:21:11,  4.33s/it]
 30%|█████████████████████████████████▎                                                                            | 1208/3996 [1:31:50<3:19:10,  4.29s/it]
 30%|█████████████████████████████████▎                                                                            | 1209/3996 [1:31:54<3:17:52,  4.26s/it]
 30%|█████████████████████████████████▎                                                                            | 1210/3996 [1:31:58<3:17:05,  4.24s/it]
 30%|█████████████████████████████████▎                                                                            | 1211/3996 [1:32:02<3:16:27,  4.23s/it]
 30%|█████████████████████████████████▎                                                                            | 1212/3996 [1:32:07<3:16:03,  4.23s/it]
 30%|█████████████████████████████████▍                                                                            | 1213/3996 [1:32:11<3:23:09,  4.38s/it]
 30%|█████████████████████████████████▍                                                                            | 1214/3996 [1:32:16<3:20:49,  4.33s/it]
 30%|█████████████████████████████████▍                                                                            | 1215/3996 [1:32:20<3:18:51,  4.29s/it]
 30%|█████████████████████████████████▍                                                                            | 1216/3996 [1:32:24<3:17:23,  4.26s/it]
 30%|█████████████████████████████████▌                                                                            | 1217/3996 [1:32:28<3:16:39,  4.25s/it]
 30%|█████████████████████████████████▌                                                                            | 1218/3996 [1:32:32<3:15:45,  4.23s/it]
 31%|█████████████████████████████████▌                                                                            | 1219/3996 [1:32:37<3:15:16,  4.22s/it]
 31%|█████████████████████████████████▌                                                                            | 1220/3996 [1:32:41<3:22:42,  4.38s/it]
 31%|█████████████████████████████████▌                                                                            | 1221/3996 [1:32:46<3:20:24,  4.33s/it]
 31%|█████████████████████████████████▋                                                                            | 1222/3996 [1:32:50<3:18:23,  4.29s/it]
 31%|█████████████████████████████████▋                                                                            | 1223/3996 [1:32:54<3:17:06,  4.26s/it]
 31%|█████████████████████████████████▋                                                                            | 1224/3996 [1:32:58<3:16:20,  4.25s/it]
 31%|█████████████████████████████████▋                                                                            | 1225/3996 [1:33:02<3:15:29,  4.23s/it]
                                                                                                                                                           
{'loss': 0.5533, 'grad_norm': 0.18718039989471436, 'learning_rate': 0.00016166212130956382, 'ppl': 1.739, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4211.64, 'total_tokens': 24415919, 'epoch': 0.92}
+
 31%|█████████████████████████████████▋                                                                            | 1225/3996 [1:33:02<3:15:29,  4.23s/it]
 31%|█████████████████████████████████▋                                                                            | 1226/3996 [1:33:07<3:14:55,  4.22s/it]
 31%|█████████████████████████████████▊                                                                            | 1227/3996 [1:33:11<3:22:07,  4.38s/it]
 31%|█████████████████████████████████▊                                                                            | 1228/3996 [1:33:16<3:19:43,  4.33s/it]
 31%|█████████████████████████████████▊                                                                            | 1229/3996 [1:33:20<3:17:37,  4.29s/it]
 31%|█████████████████████████████████▊                                                                            | 1230/3996 [1:33:24<3:16:05,  4.25s/it]
 31%|█████████████████████████████████▉                                                                            | 1231/3996 [1:33:28<3:15:22,  4.24s/it]
 31%|█████████████████████████████████▉                                                                            | 1232/3996 [1:33:32<3:14:38,  4.23s/it]
 31%|█████████████████████████████████▉                                                                            | 1233/3996 [1:33:36<3:14:11,  4.22s/it]
 31%|█████████████████████████████████▉                                                                            | 1234/3996 [1:33:41<3:21:24,  4.38s/it]
 31%|█████████████████████████████████▉                                                                            | 1235/3996 [1:33:45<3:18:51,  4.32s/it]
 31%|██████████████████████████████████                                                                            | 1236/3996 [1:33:50<3:17:08,  4.29s/it]
 31%|██████████████████████████████████                                                                            | 1237/3996 [1:33:54<3:15:39,  4.26s/it]
 31%|██████████████████████████████████                                                                            | 1238/3996 [1:33:58<3:14:53,  4.24s/it]
 31%|██████████████████████████████████                                                                            | 1239/3996 [1:34:02<3:14:03,  4.22s/it]
 31%|██████████████████████████████████▏                                                                           | 1240/3996 [1:34:06<3:13:33,  4.21s/it]
 31%|██████████████████████████████████▏                                                                           | 1241/3996 [1:34:11<3:20:44,  4.37s/it]
 31%|██████████████████████████████████▏                                                                           | 1242/3996 [1:34:15<3:18:30,  4.32s/it]
 31%|██████████████████████████████████▏                                                                           | 1243/3996 [1:34:20<3:16:43,  4.29s/it]
 31%|██████████████████████████████████▏                                                                           | 1244/3996 [1:34:24<3:15:21,  4.26s/it]
 31%|██████████████████████████████████▎                                                                           | 1245/3996 [1:34:28<3:14:31,  4.24s/it]
 31%|██████████████████████████████████▎                                                                           | 1246/3996 [1:34:32<3:13:41,  4.23s/it]
 31%|██████████████████████████████████▎                                                                           | 1247/3996 [1:34:36<3:13:12,  4.22s/it]
 31%|██████████████████████████████████▎                                                                           | 1248/3996 [1:34:41<3:20:30,  4.38s/it]
 31%|██████████████████████████████████▍                                                                           | 1249/3996 [1:34:45<3:18:12,  4.33s/it]
 31%|██████████████████████████████████▍                                                                           | 1250/3996 [1:34:49<3:16:15,  4.29s/it]
                                                                                                                                                           
{'loss': 0.5492, 'grad_norm': 0.17105573415756226, 'learning_rate': 0.0001600626553051268, 'ppl': 1.7319, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4183.86, 'total_tokens': 24854345, 'epoch': 0.94}
+
 31%|██████████████████████████████████▍                                                                           | 1250/3996 [1:34:49<3:16:15,  4.29s/it]
 31%|██████████████████████████████████▍                                                                           | 1251/3996 [1:34:54<3:14:54,  4.26s/it]
 31%|██████████████████████████████████▍                                                                           | 1252/3996 [1:34:58<3:14:08,  4.25s/it]
 31%|██████████████████████████████████▍                                                                           | 1253/3996 [1:35:02<3:13:32,  4.23s/it]
 31%|██████████████████████████████████▌                                                                           | 1254/3996 [1:35:06<3:12:53,  4.22s/it]
 31%|██████████████████████████████████▌                                                                           | 1255/3996 [1:35:11<3:25:54,  4.51s/it]
 31%|██████████████████████████████████▌                                                                           | 1256/3996 [1:35:16<3:21:37,  4.42s/it]
 31%|█████████���████████████████████████▌                                                                           | 1257/3996 [1:35:20<3:18:30,  4.35s/it]
 31%|██████████████████████████████████▋                                                                           | 1258/3996 [1:35:24<3:16:02,  4.30s/it]
 32%|██████████████████████████████████▋                                                                           | 1259/3996 [1:35:28<3:14:50,  4.27s/it]
 32%|██████████████████████████████████▋                                                                           | 1260/3996 [1:35:32<3:13:34,  4.25s/it]
 32%|██████████████████████████████████▋                                                                           | 1261/3996 [1:35:37<3:13:04,  4.24s/it]
 32%|██████████████████████████████████▋                                                                           | 1262/3996 [1:35:41<3:19:50,  4.39s/it]
 32%|██████████████████████████████████▊                                                                           | 1263/3996 [1:35:46<3:17:12,  4.33s/it]
 32%|██████████████████████████████████▊                                                                           | 1264/3996 [1:35:50<3:15:07,  4.29s/it]
 32%|██████████████████████████████████▊                                                                           | 1265/3996 [1:35:54<3:13:48,  4.26s/it]
 32%|██████████████████████████████████▊                                                                           | 1266/3996 [1:35:58<3:13:03,  4.24s/it]
 32%|██████████████████████████████████▉                                                                           | 1267/3996 [1:36:02<3:12:19,  4.23s/it]
 32%|██████████████████████████████████▉                                                                           | 1268/3996 [1:36:07<3:11:43,  4.22s/it]
 32%|██████████████████████████████████▉                                                                           | 1269/3996 [1:36:11<3:18:31,  4.37s/it]
 32%|██████████████████████████████████▉                                                                           | 1270/3996 [1:36:15<3:16:08,  4.32s/it]
 32%|██████████████████████████████████▉                                                                           | 1271/3996 [1:36:20<3:14:19,  4.28s/it]
 32%|███████████████████████████████████                                                                           | 1272/3996 [1:36:24<3:12:47,  4.25s/it]
 32%|███████████████████████████████████                                                                           | 1273/3996 [1:36:28<3:12:04,  4.23s/it]
 32%|███████████████████████████████████                                                                           | 1274/3996 [1:36:32<3:11:19,  4.22s/it]
 32%|███████████████████████████████████                                                                           | 1275/3996 [1:36:36<3:11:02,  4.21s/it]
                                                                                                                                                           
{'loss': 0.5348, 'grad_norm': 0.1733955442905426, 'learning_rate': 0.0001584387813292454, 'ppl': 1.7071, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4172.93, 'total_tokens': 25292647, 'epoch': 0.96}
+
 32%|███████████████████████████████████                                                                           | 1275/3996 [1:36:36<3:11:02,  4.21s/it]
 32%|███████████████████████████████████▏                                                                          | 1276/3996 [1:36:41<3:18:12,  4.37s/it]
 32%|███████████████████████████████████▏                                                                          | 1277/3996 [1:36:45<3:15:55,  4.32s/it]
 32%|███████████████████████████████████▏                                                                          | 1278/3996 [1:36:50<3:14:04,  4.28s/it]
 32%|███████████████████████████████████▏                                                                          | 1279/3996 [1:36:54<3:12:28,  4.25s/it]
 32%|███████████████████████████████████▏                                                                          | 1280/3996 [1:36:58<3:11:55,  4.24s/it]
 32%|███████████████████████████████████▎                                                                          | 1281/3996 [1:37:02<3:11:19,  4.23s/it]
 32%|███████████████████████████████████▎                                                                          | 1282/3996 [1:37:06<3:10:46,  4.22s/it]
 32%|███████████████████████████████████▎                                                                          | 1283/3996 [1:37:11<3:17:40,  4.37s/it]
 32%|███████████████████████████████████▎                                                                          | 1284/3996 [1:37:15<3:15:15,  4.32s/it]
 32%|███████████████████████████████████▎                                                                          | 1285/3996 [1:37:19<3:13:27,  4.28s/it]
 32%|███████████████████████████████████▍                                                                          | 1286/3996 [1:37:24<3:12:00,  4.25s/it]
 32%|███████████████████████████████████▍                                                                          | 1287/3996 [1:37:28<3:11:07,  4.23s/it]
 32%|███████████████████████████████████▍                                                                          | 1288/3996 [1:37:32<3:10:23,  4.22s/it]
 32%|███████████████████████████████████▍                                                                          | 1289/3996 [1:37:36<3:09:58,  4.21s/it]
 32%|███████████████████████████████████▌                                                                          | 1290/3996 [1:37:41<3:16:55,  4.37s/it]
 32%|███████████████████████████████████▌                                                                          | 1291/3996 [1:37:45<3:14:49,  4.32s/it]
 32%|███████████████████████████████████▌                                                                          | 1292/3996 [1:37:49<3:12:46,  4.28s/it]
 32%|███████████████████████████████████▌                                                                          | 1293/3996 [1:37:53<3:11:21,  4.25s/it]
 32%|███████████████████████████████████▌                                                                          | 1294/3996 [1:37:58<3:10:40,  4.23s/it]
 32%|███████████████████████████████████▋                                                                          | 1295/3996 [1:38:02<3:10:04,  4.22s/it]
 32%|███████████████████████████████████▋                                                                          | 1296/3996 [1:38:06<3:09:44,  4.22s/it]
 32%|███████████████████████████████████▋                                                                          | 1297/3996 [1:38:11<3:16:27,  4.37s/it]
 32%|███████████████████████████████████▋                                                                          | 1298/3996 [1:38:15<3:14:08,  4.32s/it]
 33%|███████████████████████████████████▊                                                                          | 1299/3996 [1:38:19<3:12:19,  4.28s/it]
 33%|███████████████████████████████████▊                                                                          | 1300/3996 [1:38:23<3:10:50,  4.25s/it]
                                                                                                                                                           
{'loss': 0.5527, 'grad_norm': 0.1858205944299698, 'learning_rate': 0.00015679115928397401, 'ppl': 1.7379, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4226.34, 'total_tokens': 25733591, 'epoch': 0.98}
+
 33%|███████████████████████████████████▊                                                                          | 1300/3996 [1:38:23<3:10:50,  4.25s/it]
 33%|███████████████████████████████████▊                                                                          | 1301/3996 [1:38:28<3:10:11,  4.23s/it]
 33%|███████████████████████████████████▊                                                                          | 1302/3996 [1:38:32<3:09:18,  4.22s/it]
 33%|███████████████████████████████████▊                                                                          | 1303/3996 [1:38:36<3:08:57,  4.21s/it]
 33%|███████████████████████████████████▉                                                                          | 1304/3996 [1:38:41<3:15:50,  4.37s/it]
 33%|███████████████████████████████████▉                                                                          | 1305/3996 [1:38:45<3:13:34,  4.32s/it]
 33%|███████████████████████████████████▉                                                                          | 1306/3996 [1:38:49<3:11:38,  4.27s/it]
 33%|███████████████████████████████████▉                                                                          | 1307/3996 [1:38:53<3:10:15,  4.25s/it]
 33%|████████████████████████████████████                                                                          | 1308/3996 [1:38:57<3:09:36,  4.23s/it]
 33%|████████████████████████████████████                                                                          | 1309/3996 [1:39:02<3:09:44,  4.24s/it]
 33%|████████████████████████████████████                                                                          | 1310/3996 [1:39:06<3:13:39,  4.33s/it]
 33%|████████████████████████████████████                                                                          | 1311/3996 [1:39:11<3:18:53,  4.44s/it]
 33%|████████████████████████████████████                                                                          | 1312/3996 [1:39:15<3:15:28,  4.37s/it]
 33%|████████████████████████████████████▏                                                                         | 1313/3996 [1:39:19<3:12:56,  4.31s/it]
 33%|████████████████████████████████████▏                                                                         | 1314/3996 [1:39:23<3:11:00,  4.27s/it]
 33%|████████████████████████████████████▏                                                                         | 1315/3996 [1:39:28<3:09:56,  4.25s/it]
 33%|████████████████████████████████████▏                                                                         | 1316/3996 [1:39:32<3:08:55,  4.23s/it]
 33%|████████████████████████████████████▎                                                                         | 1317/3996 [1:39:36<3:08:27,  4.22s/it]
 33%|████████████████████████████████████▎                                                                         | 1318/3996 [1:39:41<3:15:13,  4.37s/it]
 33%|████████████████████████████████████▎                                                                         | 1319/3996 [1:39:45<3:12:37,  4.32s/it]
 33%|████████████████████████████████████▎                                                                         | 1320/3996 [1:39:49<3:10:43,  4.28s/it]
 33%|████████████████████████████████████▎                                                                         | 1321/3996 [1:39:53<3:09:14,  4.24s/it]
 33%|████████████████████████████████████▍                                                                         | 1322/3996 [1:39:58<3:08:30,  4.23s/it]
 33%|████████████████████████████████████▍                                                                         | 1323/3996 [1:40:02<3:07:59,  4.22s/it]
 33%|████████████████████████████████████▍                                                                         | 1324/3996 [1:40:06<3:07:33,  4.21s/it]
 33%|████████████████████████████████████▍                                                                         | 1325/3996 [1:40:11<3:14:14,  4.36s/it]
                                                                                                                                                           
{'loss': 0.5311, 'grad_norm': 0.1944192498922348, 'learning_rate': 0.00015512045872199276, 'ppl': 1.7008, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3655.12, 'total_tokens': 26164528, 'epoch': 0.99}
+
 33%|████████████████████████████████████▍                                                                         | 1325/3996 [1:40:11<3:14:14,  4.36s/it]
 33%|████████████████████████████████████▌                                                                         | 1326/3996 [1:40:15<3:11:51,  4.31s/it]
 33%|████████████████████████████████████▌                                                                         | 1327/3996 [1:40:19<3:10:03,  4.27s/it]
 33%|████████████████████████████████████▌                                                                         | 1328/3996 [1:40:23<3:10:21,  4.28s/it]
 33%|████████████████████████████████████▌                                                                         | 1329/3996 [1:40:28<3:09:15,  4.26s/it]
 33%|████████████████████████████████████▌                                                                         | 1330/3996 [1:40:32<3:08:03,  4.23s/it]
 33%|████████████████████████████████████▋                                                                         | 1331/3996 [1:40:36<3:07:28,  4.22s/it]
 33%|████████████████████████████████████▋                                                                         | 1332/3996 [1:40:41<3:14:14,  4.37s/it]
 33%|████████████████████████████████████▋                                                                         | 1333/3996 [1:40:45<3:12:31,  4.34s/it]
 33%|████████████████████████████████████▋                                                                         | 1334/3996 [1:40:50<3:26:13,  4.65s/it]
 33%|████████████████████████████████████▋                                                                         | 1335/3996 [1:40:54<3:20:02,  4.51s/it]
 33%|████████████████████████████████████▊                                                                         | 1336/3996 [1:40:59<3:16:03,  4.42s/it]
 33%|████████████████████████████████████▊                                                                         | 1337/3996 [1:41:03<3:13:02,  4.36s/it]
 33%|████████████████████████████████████▊                                                                         | 1338/3996 [1:41:07<3:10:46,  4.31s/it]
 34%|████████████████████████████████████▊                                                                         | 1339/3996 [1:41:12<3:16:23,  4.43s/it]
 34%|████████████████████████████████████▉                                                                         | 1340/3996 [1:41:16<3:13:17,  4.37s/it]
 34%|████████████████████████████████████▉                                                                         | 1341/3996 [1:41:20<3:10:59,  4.32s/it]
 34%|████████████████████████████████████▉                                                                         | 1342/3996 [1:41:24<3:09:17,  4.28s/it]
 34%|████████████████████████████████████▉                                                                         | 1343/3996 [1:41:29<3:08:15,  4.26s/it]
 34%|████████████████████████████████████▉                                                                         | 1344/3996 [1:41:33<3:07:18,  4.24s/it]
 34%|█████████████████████████████████████                                                                         | 1345/3996 [1:41:37<3:06:44,  4.23s/it]
 34%|█████████████████████████████████████                                                                         | 1346/3996 [1:41:42<3:19:24,  4.51s/it]
 34%|█████████████████████████████████████                                                                         | 1347/3996 [1:41:46<3:15:13,  4.42s/it]
 34%|█████████████████████████████████████                                                                         | 1348/3996 [1:41:51<3:11:59,  4.35s/it]
 34%|█████████████████████████████████████▏                                                                        | 1349/3996 [1:41:55<3:09:50,  4.30s/it]
 34%|█████████████████████████████████████▏                                                                        | 1350/3996 [1:41:59<3:08:40,  4.28s/it]
                                                                                                                                                           
{'loss': 0.5145, 'grad_norm': 0.18358173966407776, 'learning_rate': 0.00015342735857451777, 'ppl': 1.6728, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4227.25, 'total_tokens': 26610460, 'epoch': 1.01}
+
 34%|█████████████████████████████████████▏                                                                        | 1350/3996 [1:41:59<3:08:40,  4.28s/it]
 34%|█████████████████████████████████████▏                                                                        | 1351/3996 [1:42:03<3:07:23,  4.25s/it]
 34%|█████████████████████████████████████▏                                                                        | 1352/3996 [1:42:07<3:06:42,  4.24s/it]
 34%|█████████████████████████████████████▏                                                                        | 1353/3996 [1:42:12<3:13:00,  4.38s/it]
 34%|█████████████████████████████████████▎                                                                        | 1354/3996 [1:42:16<3:10:31,  4.33s/it]
 34%|█████████████████████████████████████▎                                                                        | 1355/3996 [1:42:20<3:08:51,  4.29s/it]
 34%|█████████████████████████████████████▎                                                                        | 1356/3996 [1:42:25<3:07:29,  4.26s/it]
 34%|█████████████████████████████████████▎                                                                        | 1357/3996 [1:42:29<3:06:46,  4.25s/it]
 34%|█████████████████████████████████████▍                                                                        | 1358/3996 [1:42:33<3:06:06,  4.23s/it]
 34%|█████████████████████████████████████▍                                                                        | 1359/3996 [1:42:37<3:05:42,  4.23s/it]
 34%|█████████████████████████████████████▍                                                                        | 1360/3996 [1:42:42<3:12:26,  4.38s/it]
 34%|█████████████████████████████████████▍                                                                        | 1361/3996 [1:42:46<3:10:06,  4.33s/it]
 34%|█████████████████████████████████████▍                                                                        | 1362/3996 [1:42:50<3:08:08,  4.29s/it]
 34%|█████████████████████████████████████▌                                                                        | 1363/3996 [1:42:55<3:06:49,  4.26s/it]
 34%|█████████████████████████████████████▌                                                                        | 1364/3996 [1:42:59<3:11:07,  4.36s/it]
 34%|█████████████████████████████████████▌                                                                        | 1365/3996 [1:43:03<3:08:58,  4.31s/it]
 34%|█████████████████████████████████████▌                                                                        | 1366/3996 [1:43:08<3:07:38,  4.28s/it]
 34%|█████████████████████████████████████▋                                                                        | 1367/3996 [1:43:12<3:13:35,  4.42s/it]
 34%|█████████████████████████████████████▋                                                                        | 1368/3996 [1:43:17<3:10:34,  4.35s/it]
 34%|█████████████████████████████████████▋                                                                        | 1369/3996 [1:43:21<3:08:28,  4.30s/it]
 34%|█████████████████████████████████████▋                                                                        | 1370/3996 [1:43:25<3:06:50,  4.27s/it]
 34%|█████████████████████████████████████▋                                                                        | 1371/3996 [1:43:29<3:05:56,  4.25s/it]
 34%|█████████████████████████████████████▊                                                                        | 1372/3996 [1:43:33<3:05:04,  4.23s/it]
 34%|█████████████████████████████████████▊                                                                        | 1373/3996 [1:43:38<3:04:39,  4.22s/it]
 34%|█████████████████████████████████████▊                                                                        | 1374/3996 [1:43:42<3:11:21,  4.38s/it]
 34%|█████████████████████████████████████▊                                                                        | 1375/3996 [1:43:46<3:08:57,  4.33s/it]
                                                                                                                                                           
{'loss': 0.5081, 'grad_norm': 0.1853465735912323, 'learning_rate': 0.00015171254687540038, 'ppl': 1.6621, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4318.88, 'total_tokens': 27064008, 'epoch': 1.03}
+
 34%|█████████████████████████████████████▊                                                                        | 1375/3996 [1:43:46<3:08:57,  4.33s/it]
 34%|██████████████████��██████████████████▉                                                                        | 1376/3996 [1:43:51<3:07:07,  4.29s/it]
 34%|█████████████████████████████████████▉                                                                        | 1377/3996 [1:43:55<3:05:44,  4.26s/it]
 34%|█████████████████████████████████████▉                                                                        | 1378/3996 [1:43:59<3:05:09,  4.24s/it]
 35%|█████████████████████████████████████▉                                                                        | 1379/3996 [1:44:03<3:05:10,  4.25s/it]
 35%|█████████████████████████████████████▉                                                                        | 1380/3996 [1:44:08<3:04:35,  4.23s/it]
 35%|██████████████████████████████████████                                                                        | 1381/3996 [1:44:12<3:12:50,  4.42s/it]
 35%|██████████████████████████████████████                                                                        | 1382/3996 [1:44:17<3:09:44,  4.36s/it]
 35%|██████████████████████████████████████                                                                        | 1383/3996 [1:44:21<3:07:29,  4.31s/it]
 35%|██████████████████████████████████████                                                                        | 1384/3996 [1:44:25<3:05:57,  4.27s/it]
 35%|██████████████████████████████████████▏                                                                       | 1385/3996 [1:44:29<3:05:07,  4.25s/it]
 35%|██████████████████████████████████████▏                                                                       | 1386/3996 [1:44:33<3:04:20,  4.24s/it]
 35%|██████████████████████████████████████▏                                                                       | 1387/3996 [1:44:38<3:03:53,  4.23s/it]
 35%|██████████████████████████████████████▏                                                                       | 1388/3996 [1:44:42<3:10:35,  4.38s/it]
 35%|██████████████████████████████████████▏                                                                       | 1389/3996 [1:44:47<3:08:13,  4.33s/it]
 35%|██████████████████████████████████████▎                                                                       | 1390/3996 [1:44:51<3:06:25,  4.29s/it]
 35%|██████████████████████████████████████▎                                                                       | 1391/3996 [1:44:55<3:05:06,  4.26s/it]
 35%|██████████████████████████████████████▎                                                                       | 1392/3996 [1:44:59<3:04:15,  4.25s/it]
 35%|██████████████████████████████████████▎                                                                       | 1393/3996 [1:45:03<3:03:31,  4.23s/it]
 35%|██████████████████████████████████████▎                                                                       | 1394/3996 [1:45:08<3:03:07,  4.22s/it]
 35%|██████████████████████████████████████▍                                                                       | 1395/3996 [1:45:12<3:09:46,  4.38s/it]
 35%|██████████████████████████████████████▍                                                                       | 1396/3996 [1:45:16<3:07:24,  4.32s/it]
 35%|██████████████████████████████████████▍                                                                       | 1397/3996 [1:45:21<3:05:44,  4.29s/it]
 35%|███████████████���██████████████████████▍                                                                       | 1398/3996 [1:45:25<3:04:20,  4.26s/it]
 35%|██████████████████████████████████████▌                                                                       | 1399/3996 [1:45:29<3:03:36,  4.24s/it]
 35%|██████████████████████████████████████▌                                                                       | 1400/3996 [1:45:33<3:02:50,  4.23s/it]
                                                                                                                                                           
{'loss': 0.5185, 'grad_norm': 0.18925060331821442, 'learning_rate': 0.0001499767204815273, 'ppl': 1.6795, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4324.01, 'total_tokens': 27516590, 'epoch': 1.05}
+
 35%|██████████████████████████████████████▌                                                                       | 1400/3996 [1:45:33<3:02:50,  4.23s/it]
 35%|██████████████████████████████████████▌                                                                       | 1401/3996 [1:45:37<3:02:25,  4.22s/it]
 35%|██████████████████████████████████████▌                                                                       | 1402/3996 [1:45:42<3:09:12,  4.38s/it]
 35%|██████████████████████████████████████▌                                                                       | 1403/3996 [1:45:46<3:07:30,  4.34s/it]
 35%|██████████████████████████████████████▋                                                                       | 1404/3996 [1:45:51<3:05:18,  4.29s/it]
 35%|██████████████████████████████████████▋                                                                       | 1405/3996 [1:45:55<3:03:56,  4.26s/it]
 35%|██████████████████████████████████████▋                                                                       | 1406/3996 [1:45:59<3:03:17,  4.25s/it]
 35%|██████████████████████████████████████▋                                                                       | 1407/3996 [1:46:03<3:02:28,  4.23s/it]
 35%|██████████████████████████████████████▊                                                                       | 1408/3996 [1:46:07<3:01:55,  4.22s/it]
 35%|██████████████████████████████████████▊                                                                       | 1409/3996 [1:46:12<3:08:19,  4.37s/it]
 35%|██████████████████████████████████████▊                                                                       | 1410/3996 [1:46:16<3:06:06,  4.32s/it]
 35%|██████████████████████████████████████▊                                                                       | 1411/3996 [1:46:21<3:04:27,  4.28s/it]
 35%|██████████████████████████████████████▊                                                                       | 1412/3996 [1:46:25<3:03:10,  4.25s/it]
 35%|██████████████████████████████████████▉                                                                       | 1413/3996 [1:46:29<3:02:35,  4.24s/it]
 35%|██████████████████████████████████████▉                                                                       | 1414/3996 [1:46:33<3:01:47,  4.22s/it]
 35%|██████████████████████████████████████▉                                                                       | 1415/3996 [1:46:37<3:01:23,  4.22s/it]
 35%|██████████████████████████████████████▉                                                                       | 1416/3996 [1:46:42<3:08:04,  4.37s/it]
 35%|██████████��████████████████████████████                                                                       | 1417/3996 [1:46:46<3:05:53,  4.32s/it]
 35%|███████████████████████████████████████                                                                       | 1418/3996 [1:46:51<3:05:40,  4.32s/it]
 36%|███████████████████████████████████████                                                                       | 1419/3996 [1:46:55<3:03:47,  4.28s/it]
 36%|███████████████████████████████████████                                                                       | 1420/3996 [1:46:59<3:02:43,  4.26s/it]
 36%|███████████████████████████████████████                                                                       | 1421/3996 [1:47:03<3:02:04,  4.24s/it]
 36%|███████████████████████████████████████▏                                                                      | 1422/3996 [1:47:07<3:01:27,  4.23s/it]
 36%|███████████████████████████████████████▏                                                                      | 1423/3996 [1:47:12<3:07:57,  4.38s/it]
 36%|███████████████████████████████████████▏                                                                      | 1424/3996 [1:47:16<3:05:40,  4.33s/it]
 36%|███████████████████████████████████████▏                                                                      | 1425/3996 [1:47:21<3:03:54,  4.29s/it]
                                                                                                                                                           
{'loss': 0.5234, 'grad_norm': 0.20961470901966095, 'learning_rate': 0.00014822058478963532, 'ppl': 1.6878, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4319.64, 'total_tokens': 27970075, 'epoch': 1.07}
+
 36%|███████████████████████████████████████▏                                                                      | 1425/3996 [1:47:21<3:03:54,  4.29s/it]
 36%|███████████████████████████████████████▎                                                                      | 1426/3996 [1:47:25<3:03:20,  4.28s/it]
 36%|███████████████████████████████████████▎                                                                      | 1427/3996 [1:47:29<3:02:25,  4.26s/it]
 36%|███████████████████████████████████████▎                                                                      | 1428/3996 [1:47:33<3:01:33,  4.24s/it]
 36%|███████████████████████████████████████▎                                                                      | 1429/3996 [1:47:37<3:01:00,  4.23s/it]
 36%|███████████████████████████████████████▎                                                                      | 1430/3996 [1:47:42<3:07:38,  4.39s/it]
 36%|███████████████████████████████████████▍                                                                      | 1431/3996 [1:47:46<3:05:15,  4.33s/it]
 36%|███████████████████████████████████████▍                                                                      | 1432/3996 [1:47:51<3:03:32,  4.30s/it]
 36%|███████████████████████████████████████▍                                                                      | 1433/3996 [1:47:55<3:02:14,  4.27s/it]
 36%|███████████████████████████████████████▍                                                                      | 1434/3996 [1:47:59<3:01:28,  4.25s/it]
 36%|███████████████████████████████████████▌                                                                      | 1435/3996 [1:48:03<3:00:52,  4.24s/it]
 36%|███████████████████████████████████████▌                                                                      | 1436/3996 [1:48:07<3:00:20,  4.23s/it]
 36%|███████████████████████████████████████▌                                                                      | 1437/3996 [1:48:12<3:06:38,  4.38s/it]
 36%|███████████████████████████████████████▌                                                                      | 1438/3996 [1:48:16<3:04:26,  4.33s/it]
 36%|███████████████████████████████████████▌                                                                      | 1439/3996 [1:48:21<3:02:52,  4.29s/it]
 36%|███████████████████████████████████████▋                                                                      | 1440/3996 [1:48:25<3:01:31,  4.26s/it]
 36%|███████████████████████████████████████▋                                                                      | 1441/3996 [1:48:29<3:00:49,  4.25s/it]
 36%|███████████████████████████████████████▋                                                                      | 1442/3996 [1:48:33<3:00:11,  4.23s/it]
 36%|███████████████████████████████████████▋                                                                      | 1443/3996 [1:48:37<2:59:44,  4.22s/it]
 36%|███████████████████████████████████████▋                                                                      | 1444/3996 [1:48:42<3:06:12,  4.38s/it]
 36%|███████████████████████████████████████▊                                                                      | 1445/3996 [1:48:46<3:03:56,  4.33s/it]
 36%|███████████████████████████████████████▊                                                                      | 1446/3996 [1:48:51<3:03:50,  4.33s/it]
 36%|███████████████████████████████████████▊                                                                      | 1447/3996 [1:48:55<3:02:04,  4.29s/it]
 36%|███████████████████████████████████████▊                                                                      | 1448/3996 [1:48:59<3:01:01,  4.26s/it]
 36%|███████████████████████████████████████▉                                                                      | 1449/3996 [1:49:03<3:00:05,  4.24s/it]
 36%|███████████████████████████████████████▉                                                                      | 1450/3996 [1:49:07<2:59:40,  4.23s/it]
                                                                                                                                                           
{'loss': 0.5169, 'grad_norm': 0.1982697695493698, 'learning_rate': 0.0001464448534496555, 'ppl': 1.6768, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4267.88, 'total_tokens': 28419716, 'epoch': 1.09}
+
 36%|███████████████████████████████████████▉                                                                      | 1450/3996 [1:49:07<2:59:40,  4.23s/it]
 36%|███████████████████████████████████████▉                                                                      | 1451/3996 [1:49:12<3:05:57,  4.38s/it]
 36%|███████████████████████████████████████▉                                                                      | 1452/3996 [1:49:16<3:03:39,  4.33s/it]
 36%|███████████████████████████████████████▉                                                                      | 1453/3996 [1:49:21<3:01:58,  4.29s/it]
 36%|████████████████████████████████████████                                                                      | 1454/3996 [1:49:25<3:04:13,  4.35s/it]
 36%|████████████████████████████████████████                                                                      | 1455/3996 [1:49:29<3:02:19,  4.31s/it]
 36%|████████████████████████████████████████                                                                      | 1456/3996 [1:49:33<3:00:52,  4.27s/it]
 36%|████████████████████████████████████████                                                                      | 1457/3996 [1:49:38<2:59:54,  4.25s/it]
 36%|████████████████████████████████████████▏                                                                     | 1458/3996 [1:49:42<3:06:09,  4.40s/it]
 37%|████████████████████████████████████████▏                                                                     | 1459/3996 [1:49:47<3:03:37,  4.34s/it]
 37%|████████████████████████████████████████▏                                                                     | 1460/3996 [1:49:51<3:02:10,  4.31s/it]
 37%|████████████████████████████████████████▏                                                                     | 1461/3996 [1:49:55<3:02:15,  4.31s/it]
 37%|████████████████████████████████████████▏                                                                     | 1462/3996 [1:49:59<3:00:47,  4.28s/it]
 37%|████████████████████████████████████████▎                                                                     | 1463/3996 [1:50:04<2:59:45,  4.26s/it]
 37%|████████████████████████████████████████▎                                                                     | 1464/3996 [1:50:08<2:59:04,  4.24s/it]
 37%|████████████████████████████████████████▎                                                                     | 1465/3996 [1:50:13<3:05:16,  4.39s/it]
 37%|████████████████████████████████████████▎                                                                     | 1466/3996 [1:50:17<3:02:50,  4.34s/it]
 37%|████████████████████████████████████████▍                                                                     | 1467/3996 [1:50:21<3:01:03,  4.30s/it]
 37%|████████████████████████████████████████▍                                                                     | 1468/3996 [1:50:25<2:59:46,  4.27s/it]
 37%|████████████████████████████████████████▍                                                                     | 1469/3996 [1:50:29<2:58:50,  4.25s/it]
 37%|████████████████████████████████████████▍                                                                     | 1470/3996 [1:50:34<2:57:58,  4.23s/it]
 37%|████████████████████████████████████████▍                                                                     | 1471/3996 [1:50:38<2:57:33,  4.22s/it]
 37%|████████████████████████████████████████▌                                                                     | 1472/3996 [1:50:42<3:04:16,  4.38s/it]
 37%|████████████████████████████████████████▌                                                                     | 1473/3996 [1:50:47<3:02:09,  4.33s/it]
 37%|████████████████████████████████████████▌                                                                     | 1474/3996 [1:50:51<3:00:28,  4.29s/it]
 37%|████████████████████████████████████████▌                                                                     | 1475/3996 [1:50:55<2:59:05,  4.26s/it]
                                                                                                                                                           
{'loss': 0.5197, 'grad_norm': 0.1925143301486969, 'learning_rate': 0.00014465024807470376, 'ppl': 1.6815, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4264.53, 'total_tokens': 28866312, 'epoch': 1.11}
+
 37%|████████████████████████████████████████▌                                                                     | 1475/3996 [1:50:55<2:59:05,  4.26s/it]
 37%|████████████████████████████████████████▋                                                                     | 1476/3996 [1:50:59<2:58:21,  4.25s/it]
 37%|████████████████████████████████████████▋                                                                     | 1477/3996 [1:51:04<2:57:48,  4.24s/it]
 37%|████████████████████████████████████████▋                                                                     | 1478/3996 [1:51:08<2:57:15,  4.22s/it]
 37%|████████████████████████████████████████▋                                                                     | 1479/3996 [1:51:12<3:03:38,  4.38s/it]
 37%|████████████████████████████████████████▋                                                                     | 1480/3996 [1:51:17<3:01:30,  4.33s/it]
 37%|████████████████████████████████████████▊                                                                     | 1481/3996 [1:51:21<2:59:49,  4.29s/it]
 37%|████████████████████████████████████████▊                                                                     | 1482/3996 [1:51:25<2:58:33,  4.26s/it]
 37%|████████████████████████████████████████▊                                                                     | 1483/3996 [1:51:29<2:57:51,  4.25s/it]
 37%|████████████████████████████████████████▊                                                                     | 1484/3996 [1:51:33<2:57:11,  4.23s/it]
 37%|████████████████████████████████████████▉                                                                     | 1485/3996 [1:51:38<2:56:42,  4.22s/it]
 37%|████████████████████████████████████████▉                                                                     | 1486/3996 [1:51:42<3:03:17,  4.38s/it]
 37%|████████████████████████████████████████▉                                                                     | 1487/3996 [1:51:47<3:01:09,  4.33s/it]
 37%|████████████████████████████████████████▉                                                                     | 1488/3996 [1:51:51<2:59:21,  4.29s/it]
 37%|████████████████████████████████████████▉                                                                     | 1489/3996 [1:51:55<2:58:14,  4.27s/it]
 37%|█████████████████████████████████████████                                                                     | 1490/3996 [1:51:59<2:57:25,  4.25s/it]
 37%|█████████████████████████████████████████                                                                     | 1491/3996 [1:52:03<2:56:58,  4.24s/it]
 37%|█████████████████████████████████████████                                                                     | 1492/3996 [1:52:08<2:56:25,  4.23s/it]
 37%|█████████████████████████████████████████                                                                     | 1493/3996 [1:52:12<3:02:37,  4.38s/it]
 37%|█████████████████████████████████████████▏                                                                    | 1494/3996 [1:52:17<3:00:30,  4.33s/it]
 37%|█████████████████████████████████████████▏                                                                    | 1495/3996 [1:52:21<2:58:53,  4.29s/it]
 37%|█████████████████████████████████████████▏                                                                    | 1496/3996 [1:52:25<2:57:36,  4.26s/it]
 37%|█████████████████████████████████████████▏                                                                    | 1497/3996 [1:52:29<2:56:52,  4.25s/it]
 37%|█████████████████████████████████████████▏                                                                    | 1498/3996 [1:52:33<2:56:14,  4.23s/it]
 38%|█████████████████████████████████████████▎                                                                    | 1499/3996 [1:52:38<2:55:50,  4.23s/it]
 38%|█████████████████████████████████████████▎                                                                    | 1500/3996 [1:52:42<3:02:26,  4.39s/it]
                                                                                                                                                           
{'loss': 0.5204, 'grad_norm': 0.18788637220859528, 'learning_rate': 0.0001428374979478349, 'ppl': 1.6827, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3779.33, 'total_tokens': 29315968, 'epoch': 1.13}
+
 38%|█████████████████████████████████████████▎                                                                    | 1500/3996 [1:52:42<3:02:26,  4.39s/it]
 38%|█████████████████████████████████████████▎                                                                    | 1501/3996 [1:52:47<3:00:15,  4.34s/it]
 38%|█████████████████████████████████████████▎                                                                    | 1502/3996 [1:52:51<2:58:29,  4.29s/it]
 38%|█████████████████████████████████████████▎                                                                    | 1503/3996 [1:52:55<2:57:05,  4.26s/it]
 38%|█████████████████████████████████████████▍                                                                    | 1504/3996 [1:52:59<2:56:23,  4.25s/it]
 38%|█████████████████████████████████████████▍                                                                    | 1505/3996 [1:53:03<2:55:40,  4.23s/it]
 38%|█████████████████████████████████████████▍                                                                    | 1506/3996 [1:53:08<2:55:13,  4.22s/it]
 38%|█████████████████████████████████████████▍                                                                    | 1507/3996 [1:53:12<3:01:34,  4.38s/it]
 38%|█████████████████████████████████████████▌                                                                    | 1508/3996 [1:53:17<2:59:25,  4.33s/it]
 38%|█████████████████████████████████████████▌                                                                    | 1509/3996 [1:53:21<2:57:53,  4.29s/it]
 38%|█████████████████████████████████████████▌                                                                    | 1510/3996 [1:53:25<2:56:41,  4.26s/it]
 38%|█████████████████████████████████████████▌                                                                    | 1511/3996 [1:53:29<2:55:56,  4.25s/it]
 38%|█████████████████████████████████████████▌                                                                    | 1512/3996 [1:53:33<2:55:16,  4.23s/it]
 38%|█████████████████████████████████████████▋                                                                    | 1513/3996 [1:53:38<2:54:46,  4.22s/it]
 38%|█████████████████████████████████████████▋                                                                    | 1514/3996 [1:53:42<3:01:08,  4.38s/it]
 38%|█████████████████████████████████████████▋                                                                    | 1515/3996 [1:53:47<2:59:05,  4.33s/it]
 38%|█████████████████████████████████████████▋                                                                    | 1516/3996 [1:53:51<2:57:24,  4.29s/it]
 38%|█████████████████████████████████████████▊                                                                    | 1517/3996 [1:53:55<2:56:06,  4.26s/it]
 38%|█████████████████████████████████████████▊                                                                    | 1518/3996 [1:53:59<2:55:08,  4.24s/it]
 38%|█████████████████████████████████████████▊                                                                    | 1519/3996 [1:54:03<2:54:37,  4.23s/it]
 38%|█████████████████████████████████████████▊                                                                    | 1520/3996 [1:54:08<2:54:13,  4.22s/it]
 38%|█████████████████████████████████████████▊                                                                    | 1521/3996 [1:54:12<3:00:30,  4.38s/it]
 38%|█████████████████████████████████████████▉                                                                    | 1522/3996 [1:54:16<2:58:25,  4.33s/it]
 38%|█████████████████████████████████████████▉                                                                    | 1523/3996 [1:54:21<3:02:46,  4.43s/it]
 38%|█████████████████████████████████████████▉                                                                    | 1524/3996 [1:54:25<2:59:49,  4.36s/it]
 38%|█████████████████████████████████████████▉                                                                    | 1525/3996 [1:54:30<2:57:51,  4.32s/it]
                                                                                                                                                           
{'loss': 0.5164, 'grad_norm': 0.18954145908355713, 'learning_rate': 0.00014100733972568038, 'ppl': 1.676, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4282.57, 'total_tokens': 29766723, 'epoch': 1.14}
+
 38%|█████████████████████████████████████████▉                                                                    | 1525/3996 [1:54:30<2:57:51,  4.32s/it]
 38%|██████████████████████████████████████████                                                                    | 1526/3996 [1:54:34<3:00:23,  4.38s/it]
 38%|██████████████████████████████████████████                                                                    | 1527/3996 [1:54:38<2:58:11,  4.33s/it]
 38%|██████████████████████████████████████████                                                                    | 1528/3996 [1:54:43<3:03:19,  4.46s/it]
 38%|██████████████████████████████████████████                                                                    | 1529/3996 [1:54:47<3:00:04,  4.38s/it]
 38%|██████████████████████████████████████████                                                                    | 1530/3996 [1:54:51<2:57:39,  4.32s/it]
 38%|██████████████████████████████████████████▏                                                                   | 1531/3996 [1:54:56<2:56:02,  4.28s/it]
 38%|██████████████████████████████████████████▏                                                                   | 1532/3996 [1:55:00<2:55:03,  4.26s/it]
 38%|██████████████████████████████████████████▏                                                                   | 1533/3996 [1:55:04<2:54:21,  4.25s/it]
 38%|██████████████████████████████████████████▏                                                                   | 1534/3996 [1:55:08<2:53:40,  4.23s/it]
 38%|██████████████████████████████████████████▎                                                                   | 1535/3996 [1:55:13<2:59:53,  4.39s/it]
 38%|██████████████████████████████████████████▎                                                                   | 1536/3996 [1:55:17<2:57:34,  4.33s/it]
 38%|██████████████████████████████████████████▎                                                                   | 1537/3996 [1:55:21<2:55:41,  4.29s/it]
 38%|██████████████████████████████████████████▎                                                                   | 1538/3996 [1:55:26<2:54:17,  4.25s/it]
 39%|██████████████████████████████████████████▎                                                                   | 1539/3996 [1:55:30<2:53:36,  4.24s/it]
 39%|██████████████████████████████████████████▍                                                                   | 1540/3996 [1:55:34<2:52:55,  4.22s/it]
 39%|██████████████████████████████████████████▍                                                                   | 1541/3996 [1:55:38<2:52:33,  4.22s/it]
 39%|██████████████████████████████████████████▍                                                                   | 1542/3996 [1:55:43<2:58:51,  4.37s/it]
 39%|██████████████████████████████████████████▍                                                                   | 1543/3996 [1:55:47<2:56:48,  4.32s/it]
 39%|██████████████████████████████████████████▌                                                                   | 1544/3996 [1:55:52<3:00:32,  4.42s/it]
 39%|██████████████████████████████████████████▌                                                                   | 1545/3996 [1:55:56<2:57:33,  4.35s/it]
 39%|██████████████████████████████████████████▌                                                                   | 1546/3996 [1:56:00<2:55:55,  4.31s/it]
 39%|██████████████████████████████████████████▌                                                                   | 1547/3996 [1:56:04<2:54:25,  4.27s/it]
 39%|██████████████████████████████████████████▌                                                                   | 1548/3996 [1:56:09<2:53:35,  4.25s/it]
 39%|██████████████████████████████████████████▋                                                                   | 1549/3996 [1:56:13<2:59:19,  4.40s/it]
 39%|██████████████████████████████████████████▋                                                                   | 1550/3996 [1:56:17<2:56:59,  4.34s/it]
                                                                                                                                                           
{'loss': 0.5095, 'grad_norm': 0.19003146886825562, 'learning_rate': 0.00013916051713908924, 'ppl': 1.6645, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4290.76, 'total_tokens': 30218573, 'epoch': 1.16}
+
 39%|██████████████████████████████████████████▋                                                                   | 1550/3996 [1:56:17<2:56:59,  4.34s/it]
 39%|██████████████████████████████████████████▋                                                                   | 1551/3996 [1:56:22<2:55:11,  4.30s/it]
 39%|██████████████████████████████████████████▋                                                                   | 1552/3996 [1:56:26<2:53:48,  4.27s/it]
 39%|██████████████████████████████████████████▊                                                                   | 1553/3996 [1:56:30<2:53:04,  4.25s/it]
 39%|██████████████████████████████████████████▊                                                                   | 1554/3996 [1:56:34<2:52:19,  4.23s/it]
 39%|██████████████████████████████████████████▊                                                                   | 1555/3996 [1:56:38<2:51:55,  4.23s/it]
 39%|██████████████████████████████████████████▊                                                                   | 1556/3996 [1:56:43<2:58:12,  4.38s/it]
 39%|██████████████████████████████████████████▊                                                                   | 1557/3996 [1:56:47<2:55:54,  4.33s/it]
 39%|██████████████████████████████████████████▉                                                                   | 1558/3996 [1:56:52<2:54:14,  4.29s/it]
 39%|██████████████████████████████████████████▉                                                                   | 1559/3996 [1:56:56<2:52:54,  4.26s/it]
 39%|██████████████████████████████████████████▉                                                                   | 1560/3996 [1:57:00<2:52:12,  4.24s/it]
 39%|██████████████████████████████████████████▉                                                                   | 1561/3996 [1:57:04<2:51:40,  4.23s/it]
 39%|██████████████████████████████████████████▉                                                                   | 1562/3996 [1:57:08<2:51:12,  4.22s/it]
 39%|███████████████████████████████████████████                                                                   | 1563/3996 [1:57:13<2:57:16,  4.37s/it]
 39%|███████████████████████████████████████████                                                                   | 1564/3996 [1:57:17<2:55:08,  4.32s/it]
 39%|███████████████████████████████████████████                                                                   | 1565/3996 [1:57:22<2:53:34,  4.28s/it]
 39%|███████████████████████████████████████████                                                                   | 1566/3996 [1:57:26<2:52:36,  4.26s/it]
 39%|███████████████████████████████████████████▏                                                                  | 1567/3996 [1:57:30<2:51:52,  4.25s/it]
 39%|███████████████████████████████████████████▏                                                                  | 1568/3996 [1:57:34<2:51:08,  4.23s/it]
 39%|███████████████████████████████████████████▏                                                                  | 1569/3996 [1:57:38<2:50:40,  4.22s/it]
 39%|███████████████████████████████████████████▏                                                                  | 1570/3996 [1:57:43<2:56:51,  4.37s/it]
 39%|███████████████████████████████████████████▏                                                                  | 1571/3996 [1:57:47<2:54:37,  4.32s/it]
 39%|███████████████████████████████████████████▎                                                                  | 1572/3996 [1:57:51<2:53:12,  4.29s/it]
 39%|███████████████████████████████████████████▎                                                                  | 1573/3996 [1:57:56<2:51:51,  4.26s/it]
 39%|███████████████████████████████████████████▎                                                                  | 1574/3996 [1:58:00<2:51:20,  4.24s/it]
 39%|███████████████████████████████████████████▎                                                                  | 1575/3996 [1:58:04<2:50:42,  4.23s/it]
                                                                                                                                                           
{'loss': 0.522, 'grad_norm': 0.18279583752155304, 'learning_rate': 0.00013729778069089437, 'ppl': 1.6854, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4300.13, 'total_tokens': 30669810, 'epoch': 1.18}
+
 39%|███████████████████████████████████████████▎                                                                  | 1575/3996 [1:58:04<2:50:42,  4.23s/it]
 39%|███████████████████████████████████████████▍                                                                  | 1576/3996 [1:58:08<2:50:11,  4.22s/it]
 39%|███████████████████████████████████████████▍                                                                  | 1577/3996 [1:58:13<2:56:14,  4.37s/it]
 39%|███████████████████████████████████████████▍                                                                  | 1578/3996 [1:58:17<2:54:11,  4.32s/it]
 40%|███████████████████████████████████████████▍                                                                  | 1579/3996 [1:58:21<2:52:47,  4.29s/it]
 40%|███████████████████████████████████████████▍                                                                  | 1580/3996 [1:58:26<2:53:06,  4.30s/it]
 40%|███████████████████████████████████████████▌                                                                  | 1581/3996 [1:58:30<2:51:57,  4.27s/it]
 40%|███████████████████████████████████████████▌                                                                  | 1582/3996 [1:58:34<2:50:56,  4.25s/it]
 40%|███████████████████████████████████████████▌                                                                  | 1583/3996 [1:58:38<2:50:20,  4.24s/it]
 40%|███████████████████████████████████████████▌                                                                  | 1584/3996 [1:58:43<2:56:17,  4.39s/it]
 40%|███████████████████████████████████████████▋                                                                  | 1585/3996 [1:58:47<2:54:03,  4.33s/it]
 40%|███████████████████████████████████████████▋                                                                  | 1586/3996 [1:58:52<2:52:24,  4.29s/it]
 40%|███████████████████████████████████████████▋                                                                  | 1587/3996 [1:58:56<2:51:00,  4.26s/it]
 40%|███████████████████████████████████████████▋                                                                  | 1588/3996 [1:59:00<2:50:15,  4.24s/it]
 40%|███████████████████████████████████████████▋                                                                  | 1589/3996 [1:59:04<2:49:33,  4.23s/it]
 40%|███████████████████████████████████████████▊                                                                  | 1590/3996 [1:59:08<2:49:08,  4.22s/it]
 40%|███████████████████████████████████████████▊                                                                  | 1591/3996 [1:59:13<2:55:13,  4.37s/it]
 40%|███████████████████████████████████████████▊                                                                  | 1592/3996 [1:59:17<2:53:06,  4.32s/it]
 40%|███████████████████████████████████████████▊                                                                  | 1593/3996 [1:59:21<2:51:34,  4.28s/it]
 40%|███████████████████████████████████████████▉                                                                  | 1594/3996 [1:59:26<2:50:21,  4.26s/it]
 40%|███████████████████████████████████████████▉                                                                  | 1595/3996 [1:59:30<2:49:50,  4.24s/it]
 40%|███████████████████████████████████████████▉                                                                  | 1596/3996 [1:59:34<2:49:12,  4.23s/it]
 40%|███████████████████████████████████████████▉                                                                  | 1597/3996 [1:59:38<2:48:58,  4.23s/it]
 40%|███████████████████████████████████████████▉                                                                  | 1598/3996 [1:59:43<2:59:16,  4.49s/it]
 40%|████████████████████████████████████████████                                                                  | 1599/3996 [1:59:48<2:55:45,  4.40s/it]
 40%|████████████████████████████████████████████                                                                  | 1600/3996 [1:59:52<2:53:13,  4.34s/it]
                                                                                                                                                           
{'loss': 0.5003, 'grad_norm': 0.18783092498779297, 'learning_rate': 0.00013541988735092672, 'ppl': 1.6492, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4271.27, 'total_tokens': 31117586, 'epoch': 1.2}
+
 40%|████████████████████████████████████████████                                                                  | 1600/3996 [1:59:52<2:53:13,  4.34s/it]
 40%|████████████████████████████████████████████                                                                  | 1601/3996 [1:59:56<2:51:24,  4.29s/it]
 40%|████████████████████████████████████████████                                                                  | 1602/3996 [2:00:00<2:50:16,  4.27s/it]
 40%|████████████████████████████████████████████▏                                                                 | 1603/3996 [2:00:04<2:49:25,  4.25s/it]
 40%|████████████████████████████████████████████▏                                                                 | 1604/3996 [2:00:09<2:48:40,  4.23s/it]
 40%|████████████████████████████████████████████▏                                                                 | 1605/3996 [2:00:13<2:54:32,  4.38s/it]
 40%|████████████████████████████████████████████▏                                                                 | 1606/3996 [2:00:17<2:52:21,  4.33s/it]
 40%|████████████████████████████���███████████████▏                                                                 | 1607/3996 [2:00:22<2:50:38,  4.29s/it]
 40%|████████████████████████████████████████████▎                                                                 | 1608/3996 [2:00:26<2:49:25,  4.26s/it]
 40%|████████████████████████████████████████████▎                                                                 | 1609/3996 [2:00:30<2:48:49,  4.24s/it]
 40%|████████████████████████████████████████████▎                                                                 | 1610/3996 [2:00:34<2:48:11,  4.23s/it]
 40%|████████████████████████████████████████████▎                                                                 | 1611/3996 [2:00:38<2:47:50,  4.22s/it]
 40%|████████████████████████████████████████████▎                                                                 | 1612/3996 [2:00:43<2:53:57,  4.38s/it]
 40%|████████████████████████████████████████████▍                                                                 | 1613/3996 [2:00:47<2:52:29,  4.34s/it]
 40%|████████████████████████████████████████████▍                                                                 | 1614/3996 [2:00:52<2:50:45,  4.30s/it]
 40%|████████████████████████████████████████████▍                                                                 | 1615/3996 [2:00:56<2:49:15,  4.27s/it]
 40%|████████████████████████████████████████████▍                                                                 | 1616/3996 [2:01:00<2:48:38,  4.25s/it]
 40%|████████████████████████████████████████████▌                                                                 | 1617/3996 [2:01:04<2:47:50,  4.23s/it]
 40%|████████████████████████████████████████████▌                                                                 | 1618/3996 [2:01:08<2:47:16,  4.22s/it]
 41%|████████████████████████████████████████████▌                                                                 | 1619/3996 [2:01:13<2:53:20,  4.38s/it]
 41%|████████████████████████████████████████████▌                                                                 | 1620/3996 [2:01:17<2:51:12,  4.32s/it]
 41%|████████████████████████████████████████████▌                                                                 | 1621/3996 [2:01:22<2:49:43,  4.29s/it]
 41%|████████████████████████████████████████████▋                                                                 | 1622/3996 [2:01:26<2:48:20,  4.25s/it]
 41%|████████████████████████████████████████████▋                                                                 | 1623/3996 [2:01:30<2:47:46,  4.24s/it]
 41%|████████████████████████████████████████████▋                                                                 | 1624/3996 [2:01:34<2:47:16,  4.23s/it]
 41%|████████████████████████████████████████████▋                                                                 | 1625/3996 [2:01:38<2:46:40,  4.22s/it]
                                                                                                                                                           
{'loss': 0.5115, 'grad_norm': 0.199558824300766, 'learning_rate': 0.00013352760024840175, 'ppl': 1.6678, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4248.14, 'total_tokens': 31562224, 'epoch': 1.22}
+
 41%|████████████████████████████████████████████▋                                                                 | 1625/3996 [2:01:38<2:46:40,  4.22s/it]
 41%|████████████████████████████████████████████▊                                                                 | 1626/3996 [2:01:43<2:52:51,  4.38s/it]
 41%|████████████████████████████████████████████▊                                                                 | 1627/3996 [2:01:47<2:50:45,  4.32s/it]
 41%|████████████████████████████████████████████▊                                                                 | 1628/3996 [2:01:51<2:49:04,  4.28s/it]
 41%|████████████████████████████████████████████▊                                                                 | 1629/3996 [2:01:56<2:47:47,  4.25s/it]
 41%|████████████████████████████████████████████▊                                                                 | 1630/3996 [2:02:00<2:47:09,  4.24s/it]
 41%|████████████████████████████████████████████▉                                                                 | 1631/3996 [2:02:04<2:46:31,  4.22s/it]
 41%|████████████████████████████████████████████▉                                                                 | 1632/3996 [2:02:08<2:46:08,  4.22s/it]
 41%|████████████████████████████████████████████▉                                                                 | 1633/3996 [2:02:13<2:52:08,  4.37s/it]
 41%|████████████████████████████████████████████▉                                                                 | 1634/3996 [2:02:17<2:51:55,  4.37s/it]
 41%|█████████████████████████████████████████████                                                                 | 1635/3996 [2:02:22<2:49:48,  4.32s/it]
 41%|█████████████████████████████████████████████                                                                 | 1636/3996 [2:02:26<2:48:12,  4.28s/it]
 41%|█████████████████████████████████████████████                                                                 | 1637/3996 [2:02:30<2:47:21,  4.26s/it]
 41%|█████████████████████████████████████████████                                                                 | 1638/3996 [2:02:34<2:46:30,  4.24s/it]
 41%|█████████████████████████████████████████████                                                                 | 1639/3996 [2:02:38<2:45:55,  4.22s/it]
 41%|█████████████████████████████████████████████▏                                                                | 1640/3996 [2:02:43<2:51:59,  4.38s/it]
 41%|█████████████████████████████████████████████▏                                                                | 1641/3996 [2:02:47<2:49:54,  4.33s/it]
 41%|█████████████████████████████████████████████▏                                                                | 1642/3996 [2:02:51<2:48:15,  4.29s/it]
 41%|█████████████████████████████████████████████▏                                                                | 1643/3996 [2:02:56<2:47:02,  4.26s/it]
 41%|█████████████████████████████████████████████▎                                                                | 1644/3996 [2:03:00<2:46:20,  4.24s/it]
 41%|█████████████████████████████████████████████▎                                                                | 1645/3996 [2:03:04<2:45:41,  4.23s/it]
 41%|██████��██████████████████████████████████████▎                                                                | 1646/3996 [2:03:08<2:45:19,  4.22s/it]
 41%|█████████████████████████████████████████████▎                                                                | 1647/3996 [2:03:13<2:51:21,  4.38s/it]
 41%|█████████████████████████████████████████████▎                                                                | 1648/3996 [2:03:17<2:49:19,  4.33s/it]
 41%|█████████████████████████████████████████████▍                                                                | 1649/3996 [2:03:21<2:47:47,  4.29s/it]
 41%|█████████████████████████████████████████████▍                                                                | 1650/3996 [2:03:26<2:46:33,  4.26s/it]
                                                                                                                                                           
{'loss': 0.4967, 'grad_norm': 0.19465653598308563, 'learning_rate': 0.00013162168836180246, 'ppl': 1.6433, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4286.24, 'total_tokens': 32011071, 'epoch': 1.24}
+
 41%|█████████████████████████████████████████████▍                                                                | 1650/3996 [2:03:26<2:46:33,  4.26s/it]
 41%|█████████████████████████████████████████████▍                                                                | 1651/3996 [2:03:30<2:45:48,  4.24s/it]
 41%|█████████████████████████████████████████████▍                                                                | 1652/3996 [2:03:34<2:45:31,  4.24s/it]
 41%|█████████████████████████████████████████████▌                                                                | 1653/3996 [2:03:38<2:45:00,  4.23s/it]
 41%|█████████████████████████████████████████████▌                                                                | 1654/3996 [2:03:43<2:50:54,  4.38s/it]
 41%|█████████████████████████████████████████████▌                                                                | 1655/3996 [2:03:47<2:48:44,  4.32s/it]
 41%|█████████████████████████████████████████████▌                                                                | 1656/3996 [2:03:51<2:47:03,  4.28s/it]
 41%|█████████████████████████████████████████████▌                                                                | 1657/3996 [2:03:56<2:45:48,  4.25s/it]
 41%|█████████████████████████████████████████████▋                                                                | 1658/3996 [2:04:00<2:45:17,  4.24s/it]
 42%|█████████████████████████████████████████████▋                                                                | 1659/3996 [2:04:04<2:44:41,  4.23s/it]
 42%|█████████████████████████████████████████████▋                                                                | 1660/3996 [2:04:08<2:44:11,  4.22s/it]
 42%|█████████████████████████████████████████████▋                                                                | 1661/3996 [2:04:13<2:50:14,  4.37s/it]
 42%|█████████████████████████████████████████████▊                                                                | 1662/3996 [2:04:17<2:48:09,  4.32s/it]
 42%|█████████████████████████████████████████████▊                                                                | 1663/3996 [2:04:21<2:46:39,  4.29s/it]
 42%|█████████████████████████████████████████████▊                                                                | 1664/3996 [2:04:25<2:45:19,  4.25s/it]
 42%|█████████████████████████████████████████████▊                                                                | 1665/3996 [2:04:30<2:44:40,  4.24s/it]
 42%|█████████████████████████████████████████████▊                                                                | 1666/3996 [2:04:34<2:44:05,  4.23s/it]
 42%|█████████████████████████████████████████████▉                                                                | 1667/3996 [2:04:38<2:43:45,  4.22s/it]
 42%|█████████████████████████████████████████████▉                                                                | 1668/3996 [2:04:43<2:49:39,  4.37s/it]
 42%|█████████████████████████████████████████████▉                                                                | 1669/3996 [2:04:47<2:47:32,  4.32s/it]
 42%|█████████████████████████████████████████████▉                                                                | 1670/3996 [2:04:51<2:46:20,  4.29s/it]
 42%|█████████████████████████████████████████████▉                                                                | 1671/3996 [2:04:55<2:45:04,  4.26s/it]
 42%|██████████████████████████████████████████████                                                                | 1672/3996 [2:05:00<2:44:20,  4.24s/it]
 42%|██████████████████████████████████████████████                                                                | 1673/3996 [2:05:04<2:43:42,  4.23s/it]
 42%|██████████████████████████████████████████████                                                                | 1674/3996 [2:05:08<2:43:24,  4.22s/it]
 42%|██████████████████████████████████████████████                                                                | 1675/3996 [2:05:13<2:49:14,  4.37s/it]
                                                                                                                                                           
{'loss': 0.5172, 'grad_norm': 0.2054641842842102, 'learning_rate': 0.00012970292620638574, 'ppl': 1.6773, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3733.1, 'total_tokens': 32452490, 'epoch': 1.26}
+
 42%|██████████████████████████████████████████████                                                                | 1675/3996 [2:05:13<2:49:14,  4.37s/it]
 42%|██████████████████████████████████████████████▏                                                               | 1676/3996 [2:05:17<2:47:04,  4.32s/it]
 42%|██████████████████████████████████████████████▏                                                               | 1677/3996 [2:05:21<2:45:25,  4.28s/it]
 42%|██████████████████████████████████████████████▏                                                               | 1678/3996 [2:05:25<2:44:16,  4.25s/it]
 42%|██████████████████████████████████████████████▏                                                               | 1679/3996 [2:05:30<2:43:42,  4.24s/it]
 42%|██████████████████████████████████████████████▏                                                               | 1680/3996 [2:05:34<2:43:11,  4.23s/it]
 42%|██████████████████████████████████████████████▎                                                               | 1681/3996 [2:05:38<2:42:47,  4.22s/it]
 42%|██████████████████████████████████████████████▎                                                               | 1682/3996 [2:05:43<2:48:50,  4.38s/it]
 42%|██████████████████████████████████████████████▎                                                               | 1683/3996 [2:05:47<2:46:41,  4.32s/it]
 42%|██████████████████████████████████████████████▎                                                               | 1684/3996 [2:05:51<2:45:42,  4.30s/it]
 42%|██████████████████████████████████████████████▍                                                               | 1685/3996 [2:05:55<2:44:18,  4.27s/it]
 42%|██████████████████████████████████████████████▍                                                               | 1686/3996 [2:06:00<2:43:30,  4.25s/it]
 42%|██████████████████████████████████████████████▍                                                               | 1687/3996 [2:06:04<2:42:50,  4.23s/it]
 42%|██████████████████████████████████████████████▍                                                               | 1688/3996 [2:06:08<2:42:21,  4.22s/it]
 42%|██████████████████████████████████████████████▍                                                               | 1689/3996 [2:06:13<2:51:14,  4.45s/it]
 42%|██████████████████████████████████████████████▌                                                               | 1690/3996 [2:06:17<2:48:09,  4.38s/it]
 42%|██████████████████████████████████████████████▌                                                               | 1691/3996 [2:06:21<2:46:06,  4.32s/it]
 42%|██████████████████████████████████████████████▌                                                               | 1692/3996 [2:06:25<2:44:28,  4.28s/it]
 42%|██████████████████████████████████████████████▌                                                               | 1693/3996 [2:06:30<2:43:37,  4.26s/it]
 42%|██████████████████████████████████████████████▋                                                               | 1694/3996 [2:06:34<2:42:43,  4.24s/it]
 42%|██████████████████████████████████████████████▋                                                               | 1695/3996 [2:06:38<2:42:06,  4.23s/it]
 42%|██████████████████████████████████████████████▋                                                               | 1696/3996 [2:06:43<2:47:56,  4.38s/it]
 42%|██████████████████████████████████████████████▋                                                               | 1697/3996 [2:06:47<2:45:51,  4.33s/it]
 42%|██████████████████████████████████████████████▋                                                               | 1698/3996 [2:06:51<2:44:19,  4.29s/it]
 43%|██████████████████████████████████████████████▊                                                               | 1699/3996 [2:06:55<2:42:57,  4.26s/it]
 43%|██████████████████████████████████████████████▊                                                               | 1700/3996 [2:07:00<2:42:16,  4.24s/it]
                                                                                                                                                           
{'loss': 0.5149, 'grad_norm': 0.19450411200523376, 'learning_rate': 0.00012777209351943862, 'ppl': 1.6735, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4251.33, 'total_tokens': 32899103, 'epoch': 1.28}
+
 43%|██████████████████████████████████████████████▊                                                               | 1700/3996 [2:07:00<2:42:16,  4.24s/it]
 43%|██████████████████████████████████████████████▊                                                               | 1701/3996 [2:07:04<2:41:50,  4.23s/it]
 43%|██████████████████████████████████████████████▊                                                               | 1702/3996 [2:07:08<2:41:21,  4.22s/it]
 43%|██████████████████████████████████████████████▉                                                               | 1703/3996 [2:07:13<2:47:09,  4.37s/it]
 43%|██████████████████████████████████████████████▉                                                               | 1704/3996 [2:07:17<2:45:05,  4.32s/it]
 43%|██████████████████████████████████████████████▉                                                               | 1705/3996 [2:07:21<2:43:38,  4.29s/it]
 43%|██████████████████████████████████████████████▉                                                               | 1706/3996 [2:07:25<2:42:21,  4.25s/it]
 43%|██████████████████████████████████████████████▉                                                               | 1707/3996 [2:07:30<2:45:34,  4.34s/it]
 43%|███████████████████████████████████████████████                                                               | 1708/3996 [2:07:34<2:43:51,  4.30s/it]
 43%|███████████████████████████████████████████████                                                               | 1709/3996 [2:07:38<2:42:31,  4.26s/it]
 43%|███████████████████████████████████████████████                                                               | 1710/3996 [2:07:43<2:47:53,  4.41s/it]
 43%|███████████████████████████████████████████████                                                               | 1711/3996 [2:07:47<2:45:27,  4.34s/it]
 43%|███████████████████████████████████████████████▏                                                              | 1712/3996 [2:07:51<2:43:32,  4.30s/it]
 43%|███████████████████████████████████████████████▏                                                              | 1713/3996 [2:07:56<2:42:11,  4.26s/it]
 43%|███████████████████████████████████████████████▏                                                              | 1714/3996 [2:08:00<2:41:29,  4.25s/it]
 43%|███████████████████████████████████████████████▏                                                              | 1715/3996 [2:08:04<2:40:55,  4.23s/it]
 43%|███████████████████████████████████████████████▏                                                              | 1716/3996 [2:08:08<2:40:29,  4.22s/it]
 43%|███████████████████████████████████████████████▎                                                              | 1717/3996 [2:08:13<2:46:12,  4.38s/it]
 43%|███████████████████████████████████████████████▎                                                              | 1718/3996 [2:08:17<2:44:11,  4.32s/it]
 43%|███████████████████████████████████████████████▎                                                              | 1719/3996 [2:08:21<2:42:41,  4.29s/it]
 43%|███████████████████████████████████████████████▎                                                              | 1720/3996 [2:08:25<2:41:35,  4.26s/it]
 43%|███████████████████████████████████████████████▎                                                              | 1721/3996 [2:08:30<2:40:53,  4.24s/it]
 43%|███████████████████████████████████████████████▍                                                              | 1722/3996 [2:08:34<2:40:05,  4.22s/it]
 43%|███████████████████████████████████████████████▍                                                              | 1723/3996 [2:08:38<2:39:37,  4.21s/it]
 43%|███████████████████████████████████████████████▍                                                              | 1724/3996 [2:08:43<2:46:06,  4.39s/it]
 43%|███████████████████████████████████████████████▍                                                              | 1725/3996 [2:08:47<2:43:56,  4.33s/it]
                                                                                                                                                           
{'loss': 0.5205, 'grad_norm': 0.19844166934490204, 'learning_rate': 0.0001258299749434123, 'ppl': 1.6829, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4240.57, 'total_tokens': 33344569, 'epoch': 1.29}
+
 43%|███████████████████████████████████████████████▍                                                              | 1725/3996 [2:08:47<2:43:56,  4.33s/it]
 43%|███████████████████████████████████████████████▌                                                              | 1726/3996 [2:08:51<2:42:21,  4.29s/it]
 43%|███████████████████████████████████████████████▌                                                              | 1727/3996 [2:08:55<2:41:02,  4.26s/it]
 43%|███████████████████████████████████████████████▌                                                              | 1728/3996 [2:09:00<2:40:17,  4.24s/it]
 43%|███████████████████████████████████████████████▌                                                              | 1729/3996 [2:09:04<2:39:41,  4.23s/it]
 43%|███████████████████████████████████████████████▌                                                              | 1730/3996 [2:09:08<2:39:22,  4.22s/it]
 43%|███████████████████████████████████████████████▋                                                              | 1731/3996 [2:09:13<2:45:04,  4.37s/it]
 43%|███████████████████████████████████████████████▋                                                              | 1732/3996 [2:09:17<2:42:56,  4.32s/it]
 43%|███████████████████████████████████████████████▋                                                              | 1733/3996 [2:09:21<2:41:29,  4.28s/it]
 43%|███████████████████████████████████████████████▋                                                              | 1734/3996 [2:09:25<2:40:29,  4.26s/it]
 43%|███████████████████████████████████████████████▊                                                              | 1735/3996 [2:09:30<2:40:06,  4.25s/it]
 43%|███████████████████████████████████████████████▊                                                              | 1736/3996 [2:09:34<2:43:46,  4.35s/it]
 43%|█████████████████████��█████████████████████████▊                                                              | 1737/3996 [2:09:38<2:42:51,  4.33s/it]
 43%|███████████████████████████████████████████████▊                                                              | 1738/3996 [2:09:43<2:47:24,  4.45s/it]
 44%|███████████████████████████████████████████████▊                                                              | 1739/3996 [2:09:47<2:44:24,  4.37s/it]
 44%|███████████████████████████████████████████████▉                                                              | 1740/3996 [2:09:52<2:42:23,  4.32s/it]
 44%|███████████████████████████████████████████████▉                                                              | 1741/3996 [2:09:56<2:40:43,  4.28s/it]
 44%|███████████████████████████████████████████████▉                                                              | 1742/3996 [2:10:00<2:41:04,  4.29s/it]
 44%|███████████████████████████████████████████████▉                                                              | 1743/3996 [2:10:04<2:39:56,  4.26s/it]
 44%|████████████████████████████████████████████████                                                              | 1744/3996 [2:10:08<2:39:13,  4.24s/it]
 44%|████████████████████████████████████████████████                                                              | 1745/3996 [2:10:13<2:44:42,  4.39s/it]
 44%|████████████████████████████████████████████████                                                              | 1746/3996 [2:10:17<2:42:32,  4.33s/it]
 44%|████████████████████████████████████████████████                                                              | 1747/3996 [2:10:22<2:40:54,  4.29s/it]
 44%|████████████████████████████████████████████████                                                              | 1748/3996 [2:10:26<2:39:36,  4.26s/it]
 44%|████████████████████████████████████████████████▏                                                             | 1749/3996 [2:10:30<2:38:48,  4.24s/it]
 44%|████████████████████████████████████████████████▏                                                             | 1750/3996 [2:10:34<2:38:03,  4.22s/it]
                                                                                                                                                           
{'loss': 0.5033, 'grad_norm': 0.19240470230579376, 'learning_rate': 0.00012387735970706312, 'ppl': 1.6542, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4267.65, 'total_tokens': 33790426, 'epoch': 1.31}
+
 44%|████████████████████████████████████████████████▏                                                             | 1750/3996 [2:10:34<2:38:03,  4.22s/it]
 44%|████████████████████████████████████████████████▏                                                             | 1751/3996 [2:10:38<2:37:29,  4.21s/it]
 44%|████████████████████████████████████████████████▏                                                             | 1752/3996 [2:10:43<2:43:17,  4.37s/it]
 44%|████████████████████████████████████████████████▎                                                             | 1753/3996 [2:10:47<2:41:18,  4.32s/it]
 44%|████████████████████████████████████████████████▎                                                             | 1754/3996 [2:10:51<2:39:53,  4.28s/it]
 44%|████████████████████████████████████████████████▎                                                             | 1755/3996 [2:10:56<2:38:43,  4.25s/it]
 44%|████████████████████████████████████████████████▎                                                             | 1756/3996 [2:11:00<2:38:00,  4.23s/it]
 44%|████████████████████████████████████████████████▎                                                             | 1757/3996 [2:11:04<2:37:30,  4.22s/it]
 44%|████████████████████████████████████████████████▍                                                             | 1758/3996 [2:11:08<2:37:05,  4.21s/it]
 44%|████████████████████████████████████████████████▍                                                             | 1759/3996 [2:11:13<2:42:51,  4.37s/it]
 44%|████████████████████████████████████████████████▍                                                             | 1760/3996 [2:11:17<2:40:45,  4.31s/it]
 44%|████████████████████████████████████████████████▍                                                             | 1761/3996 [2:11:21<2:39:20,  4.28s/it]
 44%|████████████████████████████████████████████████▌                                                             | 1762/3996 [2:11:25<2:38:11,  4.25s/it]
 44%|████████████████████████████████████████████████▌                                                             | 1763/3996 [2:11:30<2:37:30,  4.23s/it]
 44%|████████████████████████████████████████████████▌                                                             | 1764/3996 [2:11:34<2:36:59,  4.22s/it]
 44%|████████████████████████████████████████████████▌                                                             | 1765/3996 [2:11:38<2:36:31,  4.21s/it]
 44%|████████████████████████████████████████████████▌                                                             | 1766/3996 [2:11:43<2:42:11,  4.36s/it]
 44%|████████████████████████████████████████████████▋                                                             | 1767/3996 [2:11:47<2:40:16,  4.31s/it]
 44%|████████████████████████████████████████████████▋                                                             | 1768/3996 [2:11:51<2:38:58,  4.28s/it]
 44%|████████████████████████████████████████████████▋                                                             | 1769/3996 [2:11:55<2:37:49,  4.25s/it]
 44%|████████████████████████████████████████████████▋                                                             | 1770/3996 [2:12:00<2:37:04,  4.23s/it]
 44%|████████████████████████████████████████████████▊                                                             | 1771/3996 [2:12:04<2:36:35,  4.22s/it]
 44%|████████████████████████████████████████████████▊                                                             | 1772/3996 [2:12:08<2:36:13,  4.21s/it]
 44%|████████████████████████████████████████████████▊                                                             | 1773/3996 [2:12:13<2:41:53,  4.37s/it]
 44%|████████████████████████████████████████████████▊                                                             | 1774/3996 [2:12:17<2:39:47,  4.31s/it]
 44%|████████████████████████████████████████████████▊                                                             | 1775/3996 [2:12:21<2:38:18,  4.28s/it]
                                                                                                                                                           
{'loss': 0.5103, 'grad_norm': 0.18220192193984985, 'learning_rate': 0.00012191504130472937, 'ppl': 1.6658, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4237.08, 'total_tokens': 34233908, 'epoch': 1.33}
+
 44%|████████████████████████████████████████████████▊                                                             | 1775/3996 [2:12:21<2:38:18,  4.28s/it]
 44%|████████████████████████████████████████████████▉                                                             | 1776/3996 [2:12:25<2:37:22,  4.25s/it]
 44%|████████████████████████████████████████████████▉                                                             | 1777/3996 [2:12:29<2:36:42,  4.24s/it]
 44%|████████████████████████████████████████████████▉                                                             | 1778/3996 [2:12:34<2:36:00,  4.22s/it]
 45%|████████████████████████████████████████████████▉                                                             | 1779/3996 [2:12:38<2:35:32,  4.21s/it]
 45%|████████████████████████████████████████████████▉                                                             | 1780/3996 [2:12:43<2:41:13,  4.37s/it]
 45%|█████████████████████████████████████████████████                                                             | 1781/3996 [2:12:47<2:39:09,  4.31s/it]
 45%|█████████████████████████████████████████████████                                                             | 1782/3996 [2:12:51<2:37:52,  4.28s/it]
 45%|█████████████████████████████████████████████████                                                             | 1783/3996 [2:12:55<2:36:50,  4.25s/it]
 45%|█████████████████████████████████████████████████                                                             | 1784/3996 [2:12:59<2:36:12,  4.24s/it]
 45%|█████████████████████████████████████████████████▏                                                            | 1785/3996 [2:13:04<2:35:46,  4.23s/it]
 45%|█████████████████████████████████████████████████▏                                                            | 1786/3996 [2:13:08<2:35:20,  4.22s/it]
 45%|█████████████████████████████████████████████████▏                                                            | 1787/3996 [2:13:12<2:41:00,  4.37s/it]
 45%|█████████████████████████████████████████████████▏                                                            | 1788/3996 [2:13:17<2:38:52,  4.32s/it]
 45%|█████████████████████████████████████████████████▏                                                            | 1789/3996 [2:13:21<2:37:21,  4.28s/it]
 45%|█████████████████████████████████████████████████▎                                                            | 1790/3996 [2:13:25<2:36:15,  4.25s/it]
 45%|█████████████████████████████████████████████████▎                                                            | 1791/3996 [2:13:29<2:35:45,  4.24s/it]
 45%|███████████████��█████████████████████████████████▎                                                            | 1792/3996 [2:13:33<2:35:08,  4.22s/it]
 45%|█████████████████████████████████████████████████▎                                                            | 1793/3996 [2:13:38<2:34:48,  4.22s/it]
 45%|█████████████████████████████████████████████████▍                                                            | 1794/3996 [2:13:42<2:40:27,  4.37s/it]
 45%|█████████████████████████████████████████████████▍                                                            | 1795/3996 [2:13:47<2:38:25,  4.32s/it]
 45%|█████████████████████████████████████████████████▍                                                            | 1796/3996 [2:13:51<2:36:54,  4.28s/it]
 45%|█████████████████████████████████████████████████▍                                                            | 1797/3996 [2:13:55<2:40:41,  4.38s/it]
 45%|█████████████████████████████████████████████████▍                                                            | 1798/3996 [2:14:00<2:38:31,  4.33s/it]
 45%|█████████████████████████████████████████████████▌                                                            | 1799/3996 [2:14:04<2:36:55,  4.29s/it]
 45%|█████████████████████████████████████████████████▌                                                            | 1800/3996 [2:14:08<2:35:50,  4.26s/it]
                                                                                                                                                           
{'loss': 0.5192, 'grad_norm': 0.20157551765441895, 'learning_rate': 0.00011994381717387514, 'ppl': 1.6807, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4244.09, 'total_tokens': 34678691, 'epoch': 1.35}
+
 45%|█████████████████████████████████████████████████▌                                                            | 1800/3996 [2:14:08<2:35:50,  4.26s/it]
 45%|█████████████████████████████████████████████████▌                                                            | 1801/3996 [2:14:13<2:40:55,  4.40s/it]
 45%|█████████████████████████████████████████████████▌                                                            | 1802/3996 [2:14:17<2:39:26,  4.36s/it]
 45%|█████████████████████████████████████████████████▋                                                            | 1803/3996 [2:14:21<2:37:33,  4.31s/it]
 45%|█████████████████████████████████████████████████▋                                                            | 1804/3996 [2:14:25<2:36:23,  4.28s/it]
 45%|█████████████████████████████████████████████████▋                                                            | 1805/3996 [2:14:30<2:36:03,  4.27s/it]
 45%|█████████████████████████████████████████████████▋                                                            | 1806/3996 [2:14:34<2:35:14,  4.25s/it]
 45%|█████████████████████████████████████████████████▋                                                            | 1807/3996 [2:14:38<2:34:32,  4.24s/it]
 45%|█████████████████████████████████████████████████▊                                                            | 1808/3996 [2:14:43<2:39:52,  4.38s/it]
 45%|██████████████████████████████████��██████████████▊                                                            | 1809/3996 [2:14:47<2:37:47,  4.33s/it]
 45%|█████████████████████████████████████████████████▊                                                            | 1810/3996 [2:14:51<2:36:21,  4.29s/it]
 45%|█████████████████████████████████████████████████▊                                                            | 1811/3996 [2:14:55<2:35:01,  4.26s/it]
 45%|█████████████████████████████████████████████████▉                                                            | 1812/3996 [2:15:00<2:34:22,  4.24s/it]
 45%|█████████████████████████████████████████████████▉                                                            | 1813/3996 [2:15:04<2:33:46,  4.23s/it]
 45%|█████████████████████████████████████████████████▉                                                            | 1814/3996 [2:15:08<2:33:11,  4.21s/it]
 45%|█████████████████████████████████████████████████▉                                                            | 1815/3996 [2:15:13<2:38:45,  4.37s/it]
 45%|█████████████████████████████████████████████████▉                                                            | 1816/3996 [2:15:17<2:36:45,  4.31s/it]
 45%|██████████████████████████████████████████████████                                                            | 1817/3996 [2:15:21<2:35:18,  4.28s/it]
 45%|██████████████████████████████████████████████████                                                            | 1818/3996 [2:15:25<2:34:07,  4.25s/it]
 46%|██████████████████████████████████████████████████                                                            | 1819/3996 [2:15:29<2:33:23,  4.23s/it]
 46%|██████████████████████████████████████████████████                                                            | 1820/3996 [2:15:34<2:32:47,  4.21s/it]
 46%|██████████████████████████████████████████████████▏                                                           | 1821/3996 [2:15:38<2:32:23,  4.20s/it]
 46%|██████████████████████████████████████████████████▏                                                           | 1822/3996 [2:15:42<2:37:56,  4.36s/it]
 46%|██████████████████████████████████████████████████▏                                                           | 1823/3996 [2:15:47<2:36:03,  4.31s/it]
 46%|██████████████████████████████████████████████████▏                                                           | 1824/3996 [2:15:51<2:34:36,  4.27s/it]
 46%|██████████████████████████████████████████████████▏                                                           | 1825/3996 [2:15:55<2:33:33,  4.24s/it]
                                                                                                                                                           
{'loss': 0.5011, 'grad_norm': 0.17189238965511322, 'learning_rate': 0.00011796448837103129, 'ppl': 1.6505, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4277.26, 'total_tokens': 35125624, 'epoch': 1.37}
+
 46%|██████████████████████████████████████████████████▏                                                           | 1825/3996 [2:15:55<2:33:33,  4.24s/it]
 46%|█████████████████████████████████████████████████��▎                                                           | 1826/3996 [2:15:59<2:32:55,  4.23s/it]
 46%|██████████████████████████████████████████████████▎                                                           | 1827/3996 [2:16:03<2:32:12,  4.21s/it]
 46%|██████████████████████████████████████████████████▎                                                           | 1828/3996 [2:16:08<2:31:44,  4.20s/it]
 46%|██████████████████████████████████████████████████▎                                                           | 1829/3996 [2:16:12<2:37:12,  4.35s/it]
 46%|██████████████████████████████████████████████████▍                                                           | 1830/3996 [2:16:16<2:35:22,  4.30s/it]
 46%|██████████████████████████████████████████████████▍                                                           | 1831/3996 [2:16:21<2:33:55,  4.27s/it]
 46%|██████████████████████████████████████████████████▍                                                           | 1832/3996 [2:16:25<2:32:57,  4.24s/it]
 46%|██████████████████████████████████████████████████▍                                                           | 1833/3996 [2:16:29<2:32:13,  4.22s/it]
 46%|██████████████████████████████████████████████████▍                                                           | 1834/3996 [2:16:33<2:31:34,  4.21s/it]
 46%|██████████████████████████████████████████████████▌                                                           | 1835/3996 [2:16:37<2:31:20,  4.20s/it]
 46%|██████████████████████████████████████████████████▌                                                           | 1836/3996 [2:16:42<2:36:48,  4.36s/it]
 46%|██████████████████████████████████████████████████▌                                                           | 1837/3996 [2:16:46<2:34:42,  4.30s/it]
 46%|██████████████████████████████████████████████████▌                                                           | 1838/3996 [2:16:50<2:33:20,  4.26s/it]
 46%|██████████████████████████████████████████████████▌                                                           | 1839/3996 [2:16:55<2:32:07,  4.23s/it]
 46%|██████████████████████████████████████████████████▋                                                           | 1840/3996 [2:16:59<2:31:29,  4.22s/it]
 46%|██████████████████████████████████████████████████▋                                                           | 1841/3996 [2:17:03<2:31:10,  4.21s/it]
 46%|██████████████████████████████████████████████████▋                                                           | 1842/3996 [2:17:07<2:30:52,  4.20s/it]
 46%|██████████████████████████████████████████████████▋                                                           | 1843/3996 [2:17:12<2:36:19,  4.36s/it]
 46%|██████████████████████████████████████████████████▊                                                           | 1844/3996 [2:17:16<2:34:12,  4.30s/it]
 46%|██████████████████████████████████████████████████▊                                                           | 1845/3996 [2:17:20<2:32:42,  4.26s/it]
 46%|████████████████████████████████████���█████████████▊                                                           | 1846/3996 [2:17:24<2:31:41,  4.23s/it]
 46%|██████████████████████████████████████████████████▊                                                           | 1847/3996 [2:17:29<2:31:06,  4.22s/it]
 46%|██████████████████████████████████████████████████▊                                                           | 1848/3996 [2:17:33<2:30:46,  4.21s/it]
 46%|██████████████████████████████████████████████████▉                                                           | 1849/3996 [2:17:37<2:30:24,  4.20s/it]
 46%|██████████████████████████████████████████████████▉                                                           | 1850/3996 [2:17:42<2:35:44,  4.35s/it]
                                                                                                                                                           
{'loss': 0.4994, 'grad_norm': 0.19443106651306152, 'learning_rate': 0.00011597785924626616, 'ppl': 1.6477, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3766.52, 'total_tokens': 35568850, 'epoch': 1.39}
+
 46%|██████████████████████████████████████████████████▉                                                           | 1850/3996 [2:17:42<2:35:44,  4.35s/it]
 46%|██████████████████████████████████████████████████▉                                                           | 1851/3996 [2:17:46<2:38:50,  4.44s/it]
 46%|██████████████████████████████████████████████████▉                                                           | 1852/3996 [2:17:50<2:35:48,  4.36s/it]
 46%|███████████████████████████████████████████████████                                                           | 1853/3996 [2:17:55<2:33:46,  4.31s/it]
 46%|███████████████████████████████████████████████████                                                           | 1854/3996 [2:17:59<2:32:18,  4.27s/it]
 46%|███████████████████████████████████████████████████                                                           | 1855/3996 [2:18:03<2:31:07,  4.24s/it]
 46%|███████████████████████████████████████████████████                                                           | 1856/3996 [2:18:07<2:30:25,  4.22s/it]
 46%|███████████████████████████████████████████████████                                                           | 1857/3996 [2:18:12<2:35:37,  4.37s/it]
 46%|███████████████████████████████████████████████████▏                                                          | 1858/3996 [2:18:16<2:33:41,  4.31s/it]
 47%|███████████████████████████████████████████████████▏                                                          | 1859/3996 [2:18:20<2:31:59,  4.27s/it]
 47%|███████████████████████████████████████████████████▏                                                          | 1860/3996 [2:18:24<2:30:57,  4.24s/it]
 47%|███████████████████████████████████████████████████▏                                                          | 1861/3996 [2:18:29<2:30:15,  4.22s/it]
 47%|███████████████████████████████████████████████████▎                                                          | 1862/3996 [2:18:33<2:29:32,  4.20s/it]
 47%|███████████████████████████████████████████████████▎                                                          | 1863/3996 [2:18:37<2:29:11,  4.20s/it]
 47%|███████████████████████████████████████████████████▎                                                          | 1864/3996 [2:18:42<2:34:37,  4.35s/it]
 47%|███████████████████████████████████████████████████▎                                                          | 1865/3996 [2:18:46<2:32:43,  4.30s/it]
 47%|███████████████████████████████████████████████████▎                                                          | 1866/3996 [2:18:50<2:31:18,  4.26s/it]
 47%|███████████████████████████████████████████████████▍                                                          | 1867/3996 [2:18:54<2:30:17,  4.24s/it]
 47%|███████████████████████████████████████████████████▍                                                          | 1868/3996 [2:18:58<2:29:51,  4.23s/it]
 47%|███████████████████████████████████████████████████▍                                                          | 1869/3996 [2:19:03<2:30:36,  4.25s/it]
 47%|███████████████████████████████████████████████████▍                                                          | 1870/3996 [2:19:07<2:30:38,  4.25s/it]
 47%|███████████████████████████████████████████████████▌                                                          | 1871/3996 [2:19:12<2:35:45,  4.40s/it]
 47%|███████████████████████████████████████████████████▌                                                          | 1872/3996 [2:19:16<2:33:21,  4.33s/it]
 47%|███████████████████████████████████████████████████▌                                                          | 1873/3996 [2:19:20<2:31:44,  4.29s/it]
 47%|███████████████████████████████████████████████████▌                                                          | 1874/3996 [2:19:24<2:31:32,  4.29s/it]
 47%|███████████████████████████████████████████████████▌                                                          | 1875/3996 [2:19:28<2:30:32,  4.26s/it]
                                                                                                                                                           
{'loss': 0.5083, 'grad_norm': 0.1810811311006546, 'learning_rate': 0.00011398473711631764, 'ppl': 1.6625, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4204.76, 'total_tokens': 36009980, 'epoch': 1.41}
+
 47%|███████████████████████████████████████████████████▌                                                          | 1875/3996 [2:19:28<2:30:32,  4.26s/it]
 47%|███████████████████████████████████████████████████▋                                                          | 1876/3996 [2:19:33<2:29:25,  4.23s/it]
 47%|███████████████████████████████████████████████████▋                                                          | 1877/3996 [2:19:37<2:28:54,  4.22s/it]
 47%|███████████████████████████████████████████████████▋                                                          | 1878/3996 [2:19:42<2:34:13,  4.37s/it]
 47%|███████████████████████████████████████████████████▋                                                          | 1879/3996 [2:19:46<2:32:13,  4.31s/it]
 47%|███████████████████████████████████���███████████████▊                                                          | 1880/3996 [2:19:50<2:30:46,  4.28s/it]
 47%|███████████████████████████████████████████████████▊                                                          | 1881/3996 [2:19:54<2:29:35,  4.24s/it]
 47%|███████████████████████████████████████████████████▊                                                          | 1882/3996 [2:19:58<2:28:57,  4.23s/it]
 47%|███████████████████████████████████████████████████▊                                                          | 1883/3996 [2:20:02<2:28:21,  4.21s/it]
 47%|███████████████████████████████████████████████████▊                                                          | 1884/3996 [2:20:07<2:28:07,  4.21s/it]
 47%|███████████████████████████████████████████████████▉                                                          | 1885/3996 [2:20:11<2:33:42,  4.37s/it]
 47%|███████████████████████████████████████████████████▉                                                          | 1886/3996 [2:20:16<2:31:43,  4.31s/it]
 47%|███████████████████████████████████████████████████▉                                                          | 1887/3996 [2:20:20<2:30:15,  4.27s/it]
 47%|███████████████████████████████████████████████████▉                                                          | 1888/3996 [2:20:24<2:29:13,  4.25s/it]
 47%|███████████████████████████████████████████████████▉                                                          | 1889/3996 [2:20:28<2:28:41,  4.23s/it]
 47%|████████████████████████████████████████████████████                                                          | 1890/3996 [2:20:32<2:28:13,  4.22s/it]
 47%|████████████████████████████████████████████████████                                                          | 1891/3996 [2:20:37<2:27:47,  4.21s/it]
 47%|████████████████████████████████████████████████████                                                          | 1892/3996 [2:20:41<2:32:58,  4.36s/it]
 47%|████████████████████████████████████████████████████                                                          | 1893/3996 [2:20:46<2:34:29,  4.41s/it]
 47%|████████████████████████████████████████████████████▏                                                         | 1894/3996 [2:20:50<2:32:02,  4.34s/it]
 47%|████████████████████████████████████████████████████▏                                                         | 1895/3996 [2:20:54<2:30:17,  4.29s/it]
 47%|████████████████████████████████████████████████████▏                                                         | 1896/3996 [2:20:58<2:30:57,  4.31s/it]
 47%|████████████████████████████████████████████████████▏                                                         | 1897/3996 [2:21:03<2:29:44,  4.28s/it]
 47%|████████████████████████████████████████████████████▏                                                         | 1898/3996 [2:21:07<2:28:40,  4.25s/it]
 48%|████████████████████████████████████████████████████▎                                                         | 1899/3996 [2:21:12<2:33:37,  4.40s/it]
 48%|████��███████████████████████████████████████████████▎                                                         | 1900/3996 [2:21:16<2:31:22,  4.33s/it]
                                                                                                                                                           
{'loss': 0.5141, 'grad_norm': 0.19805970788002014, 'learning_rate': 0.00011198593193651958, 'ppl': 1.6721, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4270.21, 'total_tokens': 36457032, 'epoch': 1.43}
+
 48%|████████████████████████████████████████████████████▎                                                         | 1900/3996 [2:21:16<2:31:22,  4.33s/it]
 48%|████████████████████████████████████████████████████▎                                                         | 1901/3996 [2:21:20<2:29:54,  4.29s/it]
 48%|████████████████████████████████████████████████████▎                                                         | 1902/3996 [2:21:24<2:28:46,  4.26s/it]
 48%|████████████████████████████████████████████████████▍                                                         | 1903/3996 [2:21:28<2:28:06,  4.25s/it]
 48%|████████████████████████████████████████████████████▍                                                         | 1904/3996 [2:21:33<2:27:31,  4.23s/it]
 48%|████████████████████████████████████████████████████▍                                                         | 1905/3996 [2:21:37<2:27:08,  4.22s/it]
 48%|████████████████████████████████████████████████████▍                                                         | 1906/3996 [2:21:42<2:36:35,  4.50s/it]
 48%|████████████████████████████████████████████████████▍                                                         | 1907/3996 [2:21:46<2:33:28,  4.41s/it]
 48%|████████████████████████████████████████████████████▌                                                         | 1908/3996 [2:21:50<2:31:12,  4.35s/it]
 48%|████████████████████████████████████████████████████▌                                                         | 1909/3996 [2:21:55<2:29:29,  4.30s/it]
 48%|████████████████████████████████████████████████████▌                                                         | 1910/3996 [2:21:59<2:28:26,  4.27s/it]
 48%|████████████████████████████████████████████████████▌                                                         | 1911/3996 [2:22:03<2:27:42,  4.25s/it]
 48%|████████████████████████████████████████████████████▋                                                         | 1912/3996 [2:22:07<2:27:10,  4.24s/it]
 48%|████████████████████████████████████████████████████▋                                                         | 1913/3996 [2:22:12<2:32:39,  4.40s/it]
 48%|████████████████████████████████████████████████████▋                                                         | 1914/3996 [2:22:16<2:30:25,  4.34s/it]
 48%|████████████████████████████████████████████████████▋                                                         | 1915/3996 [2:22:20<2:28:52,  4.29s/it]
 48%|████████████████████████████████████████████████████▋                                                         | 1916/3996 [2:22:24<2:27:31,  4.26s/it]
 48%|████████████████████████████████████████████████████▊                                                         | 1917/3996 [2:22:29<2:26:58,  4.24s/it]
 48%|████████████████████████████████████████████████████▊                                                         | 1918/3996 [2:22:33<2:26:21,  4.23s/it]
 48%|████████████████████████████████████████████████████▊                                                         | 1919/3996 [2:22:37<2:26:01,  4.22s/it]
 48%|████████████████████████████████████████████████████▊                                                         | 1920/3996 [2:22:42<2:31:15,  4.37s/it]
 48%|████████████████████████████████████████████████████▉                                                         | 1921/3996 [2:22:46<2:29:22,  4.32s/it]
 48%|████████████████████████████████████████████████████▉                                                         | 1922/3996 [2:22:50<2:27:50,  4.28s/it]
 48%|████████████████████████████████████████████████████▉                                                         | 1923/3996 [2:22:54<2:26:50,  4.25s/it]
 48%|████████████████████████████████████████████████████▉                                                         | 1924/3996 [2:22:59<2:29:51,  4.34s/it]
 48%|████████████████████████████████████████████████████▉                                                         | 1925/3996 [2:23:03<2:28:19,  4.30s/it]
                                                                                                                                                           
{'loss': 0.5045, 'grad_norm': 0.1936168372631073, 'learning_rate': 0.00010998225597165628, 'ppl': 1.6562, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4275.24, 'total_tokens': 36905590, 'epoch': 1.44}
+
 48%|████████████████████████████████████████████████████▉                                                         | 1925/3996 [2:23:03<2:28:19,  4.30s/it]
 48%|█████████████████████████████████████████████████████                                                         | 1926/3996 [2:23:07<2:27:18,  4.27s/it]
 48%|█████████████████████████████████████████████████████                                                         | 1927/3996 [2:23:12<2:32:09,  4.41s/it]
 48%|█████████████████████████████████████████████████████                                                         | 1928/3996 [2:23:16<2:29:56,  4.35s/it]
 48%|█████████████████████████████████████████████████████                                                         | 1929/3996 [2:23:20<2:28:14,  4.30s/it]
 48%|█████████████████████████████████████████████████████▏                                                        | 1930/3996 [2:23:25<2:26:52,  4.27s/it]
 48%|█████████████████████████████████████████████████████▏                                                        | 1931/3996 [2:23:29<2:26:07,  4.25s/it]
 48%|█████████████████████████████████████████████████████▏                                                        | 1932/3996 [2:23:33<2:25:31,  4.23s/it]
 48%|█████████████████████████████████████████████████████▏                                                        | 1933/3996 [2:23:37<2:25:11,  4.22s/it]
 48%|█████████████████████████████████████████████████████▏                                                        | 1934/3996 [2:23:42<2:30:18,  4.37s/it]
 48%|█████████████████████████████████████████████████████▎                                                        | 1935/3996 [2:23:46<2:28:32,  4.32s/it]
 48%|█████████████████████████████████████████████████████▎                                                        | 1936/3996 [2:23:50<2:27:11,  4.29s/it]
 48%|█████████████████████████████████████████████████████▎                                                        | 1937/3996 [2:23:55<2:26:07,  4.26s/it]
 48%|█████████████████████████████████████████████████████▎                                                        | 1938/3996 [2:23:59<2:25:31,  4.24s/it]
 49%|█████████████████████████████████████████████████████▍                                                        | 1939/3996 [2:24:03<2:24:59,  4.23s/it]
 49%|█████████████████████████████████████████████████████▍                                                        | 1940/3996 [2:24:07<2:24:38,  4.22s/it]
 49%|█████████████████████████████████████████████████████▍                                                        | 1941/3996 [2:24:12<2:30:16,  4.39s/it]
 49%|█████████████████████████████████████████████████████▍                                                        | 1942/3996 [2:24:16<2:28:46,  4.35s/it]
 49%|█████████████████████████████████████████████████████▍                                                        | 1943/3996 [2:24:20<2:27:07,  4.30s/it]
 49%|█████████████████████████████████████████████████████▌                                                        | 1944/3996 [2:24:25<2:25:45,  4.26s/it]
 49%|█████████████████████████████████████████████████████▌                                                        | 1945/3996 [2:24:29<2:24:56,  4.24s/it]
 49%|█████████████████████████████████████████████████████▌                                                        | 1946/3996 [2:24:33<2:24:25,  4.23s/it]
 49%|█████████████████████████████████████████████████████▌                                                        | 1947/3996 [2:24:37<2:24:01,  4.22s/it]
 49%|█████████████████████████████████████████████████████▌                                                        | 1948/3996 [2:24:42<2:29:11,  4.37s/it]
 49%|█████████████████████████████████████████████████████▋                                                        | 1949/3996 [2:24:46<2:27:25,  4.32s/it]
 49%|█████████████████████████████████████████████████████▋                                                        | 1950/3996 [2:24:50<2:26:00,  4.28s/it]
                                                                                                                                                           
{'loss': 0.5025, 'grad_norm': 0.19065748155117035, 'learning_rate': 0.00010797452346587798, 'ppl': 1.6528, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4285.81, 'total_tokens': 37354436, 'epoch': 1.46}
+
 49%|███████████████████████████���█████████████████████████▋                                                        | 1950/3996 [2:24:50<2:26:00,  4.28s/it]
 49%|█████████████████████████████████████████████████████▋                                                        | 1951/3996 [2:24:54<2:24:51,  4.25s/it]
 49%|█████████████████████████████████████████████████████▋                                                        | 1952/3996 [2:24:59<2:24:13,  4.23s/it]
 49%|█████████████████████████████████████████████████████▊                                                        | 1953/3996 [2:25:03<2:23:31,  4.22s/it]
 49%|█████████████████████████████████████████████████████▊                                                        | 1954/3996 [2:25:07<2:23:09,  4.21s/it]
 49%|█████████████████████████████████████████████████████▊                                                        | 1955/3996 [2:25:12<2:28:26,  4.36s/it]
 49%|█████████████████████████████████████████████████████▊                                                        | 1956/3996 [2:25:16<2:26:46,  4.32s/it]
 49%|█████████████████████████████████████████████████████▊                                                        | 1957/3996 [2:25:20<2:25:23,  4.28s/it]
 49%|█████████████████████████████████████████████████████▉                                                        | 1958/3996 [2:25:24<2:24:14,  4.25s/it]
 49%|█████████████████████████████████████████████████████▉                                                        | 1959/3996 [2:25:28<2:23:42,  4.23s/it]
 49%|█████████████████████████████████████████████████████▉                                                        | 1960/3996 [2:25:33<2:24:03,  4.25s/it]
 49%|█████████████████████████████████████████████████████▉                                                        | 1961/3996 [2:25:37<2:23:29,  4.23s/it]
 49%|██████████████████████████████████████████████████████                                                        | 1962/3996 [2:25:42<2:28:30,  4.38s/it]
 49%|██████████████████████████████████████████████████████                                                        | 1963/3996 [2:25:46<2:26:32,  4.32s/it]
 49%|██████████████████████████████████████████████████████                                                        | 1964/3996 [2:25:50<2:25:02,  4.28s/it]
 49%|██████████████████████████████████████████████████████                                                        | 1965/3996 [2:25:54<2:23:53,  4.25s/it]
 49%|██████████████████████████████████████████████████████                                                        | 1966/3996 [2:25:58<2:23:15,  4.23s/it]
 49%|██████████████████████████████████████████████████████▏                                                       | 1967/3996 [2:26:03<2:22:43,  4.22s/it]
 49%|██████████████████████████████████████████████████████▏                                                       | 1968/3996 [2:26:07<2:22:18,  4.21s/it]
 49%|██████████████████████████████████████████████████████▏                                                       | 1969/3996 [2:26:12<2:27:29,  4.37s/it]
 49%|██████████████████████████████████████████████████████▏                                                       | 1970/3996 [2:26:16<2:25:45,  4.32s/it]
 49%|██████████████████████████████████████████████████████▎                                                       | 1971/3996 [2:26:20<2:24:29,  4.28s/it]
 49%|██████████████████████████████████████████████████████▎                                                       | 1972/3996 [2:26:24<2:23:25,  4.25s/it]
 49%|██████████████████████████████████████████████████████▎                                                       | 1973/3996 [2:26:28<2:22:46,  4.23s/it]
 49%|██████████████████████████████████████████████████████▎                                                       | 1974/3996 [2:26:32<2:22:14,  4.22s/it]
 49%|██████████████████████████████████████████████████████▎                                                       | 1975/3996 [2:26:37<2:21:56,  4.21s/it]
                                                                                                                                                           
{'loss': 0.5102, 'grad_norm': 0.18647657334804535, 'learning_rate': 0.0001059635503118125, 'ppl': 1.6656, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4259.76, 'total_tokens': 37801500, 'epoch': 1.48}
+
 49%|██████████████████████████████████████████████████████▎                                                       | 1975/3996 [2:26:37<2:21:56,  4.21s/it]
 49%|██████████████████████████████████████████████████████▍                                                       | 1976/3996 [2:26:41<2:27:06,  4.37s/it]
 49%|██████████████████████████████████████████████████████▍                                                       | 1977/3996 [2:26:46<2:25:11,  4.31s/it]
 49%|██████████████████████████████████████████████████████▍                                                       | 1978/3996 [2:26:50<2:26:06,  4.34s/it]
 50%|██████████████████████████████████████████████████████▍                                                       | 1979/3996 [2:26:54<2:24:28,  4.30s/it]
 50%|██████████████████████████████████████████████████████▌                                                       | 1980/3996 [2:26:58<2:23:22,  4.27s/it]
 50%|██████████████████████████████████████████████████████▌                                                       | 1981/3996 [2:27:03<2:22:29,  4.24s/it]
 50%|██████████████████████████████████████████████████████▌                                                       | 1982/3996 [2:27:07<2:21:50,  4.23s/it]
 50%|██████████████████████████████████████████████████████▌                                                       | 1983/3996 [2:27:12<2:26:46,  4.37s/it]
 50%|██████████████████████████████████████████████████████▌                                                       | 1984/3996 [2:27:16<2:24:49,  4.32s/it]
 50%|██████████████████████████████████████████████████████▋                                                       | 1985/3996 [2:27:20<2:23:22,  4.28s/it]
 50%|██████████████████████████████████████████████████████▋                                                       | 1986/3996 [2:27:24<2:22:22,  4.25s/it]
 50%|██████████████████████████████████████████████████████▋                                                       | 1987/3996 [2:27:28<2:21:54,  4.24s/it]
 50%|██████████████████████████████████████████████████████▋                                                       | 1988/3996 [2:27:32<2:21:20,  4.22s/it]
 50%|██████████████████████████████████████████████████████▊                                                       | 1989/3996 [2:27:37<2:20:57,  4.21s/it]
 50%|██████████████████████████████████████████████████████▊                                                       | 1990/3996 [2:27:41<2:26:02,  4.37s/it]
 50%|██████████████████████████████████████████████████████▊                                                       | 1991/3996 [2:27:46<2:24:11,  4.32s/it]
 50%|██████████████████████████████████████████████████████▊                                                       | 1992/3996 [2:27:50<2:22:48,  4.28s/it]
 50%|██████████████████████████████████████████████████████▊                                                       | 1993/3996 [2:27:54<2:21:47,  4.25s/it]
 50%|██████████████████████████████████████████████████████▉                                                       | 1994/3996 [2:27:58<2:21:15,  4.23s/it]
 50%|██████████████████████████████████████████████████████▉                                                       | 1995/3996 [2:28:02<2:20:37,  4.22s/it]
 50%|██████████████████████████████████████████████████████▉                                                       | 1996/3996 [2:28:06<2:20:11,  4.21s/it]
 50%|██████████████████████████████████████████████████████▉                                                       | 1997/3996 [2:28:11<2:25:23,  4.36s/it]
 50%|███████████████████████████████████████████████████████                                                       | 1998/3996 [2:28:15<2:23:30,  4.31s/it]
 50%|███████████████████████████████████████████████████████                                                       | 1999/3996 [2:28:20<2:22:18,  4.28s/it]
 50%|███████████████████████████████████████████████████████                                                       | 2000/3996 [2:28:24<2:21:13,  4.25s/it]
                                                                                                                                                           
{'loss': 0.5052, 'grad_norm': 0.21211788058280945, 'learning_rate': 0.00010395015371900663, 'ppl': 1.6573, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4250.7, 'total_tokens': 38244936, 'epoch': 1.5}
+
 50%|███████████████████████████████████████████████████████                                                       | 2000/3996 [2:28:24<2:21:13,  4.25s/it][2025-12-29 05:18:09,532] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:3751] Running evaluation step...
+[2025-12-29 05:18:10,400] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.39612317085266113
+[2025-12-29 05:18:10,813] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.41332387924194336
+[2025-12-29 05:18:11,238] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.4233283996582031
+[2025-12-29 05:18:11,703] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.46494436264038086
+[2025-12-29 05:18:11,703] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:3751] gather_len_batches: [72]
 
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.41it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:54,  1.59it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<01:00,  1.43it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:05,  1.29it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:05,  1.28it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:08,  1.21it/s]
-
  9%|██████████▌                                                                                                            | 8/90 [00:06<01:07,  1.22it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:08,  1.18it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:06,  1.19it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:07,  1.17it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:05,  1.18it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:06,  1.16it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:04,  1.18it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:05,  1.15it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:12<01:03,  1.17it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:13<01:04,  1.14it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:01,  1.16it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:01,  1.15it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<00:59,  1.17it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<00:59,  1.15it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:57,  1.17it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:18<00:58,  1.15it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:19<00:56,  1.17it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<00:56,  1.15it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:54,  1.17it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:54,  1.15it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:52,  1.17it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:53,  1.15it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:24<00:51,  1.17it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:25<00:51,  1.15it/s]
-
 36%|█████████████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.17it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:50,  1.14it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:48,  1.16it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:48,  1.14it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:46,  1.15it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:47,  1.12it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:31<00:44,  1.16it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:32<00:44,  1.14it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:34<00:50,  1.01s/it]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:35<00:47,  1.04it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:42,  1.13it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:42,  1.12it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:40,  1.15it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:38<00:39,  1.14it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:39<00:37,  1.16it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:40<00:37,  1.15it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:35,  1.17it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.15it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:34,  1.17it/s]
-
 57%|██████████████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:33,  1.15it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:44<00:32,  1.17it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:45<00:32,  1.15it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:46<00:30,  1.17it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.15it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:28,  1.18it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.15it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.18it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:26,  1.15it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:51<00:25,  1.17it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:52<00:25,  1.15it/s]
-
 69%|████████████████████████████████████████████████████████████████████████████���████▎                                    | 62/90 [00:52<00:23,  1.17it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:53<00:23,  1.15it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:22,  1.17it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:21,  1.15it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:20,  1.17it/s]
-
 74%|███████████████████████████████████████████████████████████████████████████████████████▊                              | 67/90 [00:57<00:19,  1.15it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:58<00:18,  1.17it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:59<00:18,  1.15it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.17it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.15it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.17it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.15it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.17it/s]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:04<00:12,  1.16it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:05<00:11,  1.18it/s]
-
 86%|█████████████████████████████████████████████████████████████████████���██████████████████████████████▉                 | 77/90 [01:05<00:11,  1.16it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.18it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.16it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.18it/s]
-
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 81/90 [01:09<00:07,  1.15it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:10<00:06,  1.17it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:11<00:06,  1.14it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:11<00:05,  1.17it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.14it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.17it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.14it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:15<00:01,  1.17it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89/90 [01:16<00:00,  1.14it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.5417217016220093, 'eval_runtime': 79.1722, 'eval_samples_per_second': 9.22, 'eval_steps_per_second': 2.311, 'eval_ppl': 1.719, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.19, 'epoch': 0.84}
-
 28%|███████████████████████████████                                                                                | 700/2499 [1:24:46<3:08:09,  6.28s/it]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.14it/s]
-
                                                                                                                                                           
 28%|██████████████████████████████▊                                                                               | 701/2499 [1:24:52<15:51:22, 31.75s/it]
                                                                                                                                                           
{'loss': 0.565, 'grad_norm': 0.20224806666374207, 'learning_rate': 0.00016441295298774849, 'ppl': 1.7594, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4508.16, 'total_tokens': 30197925, 'epoch': 0.84}
-
 28%|██████████████████████████████▊                                                                               | 701/2499 [1:24:52<15:51:22, 31.75s/it]
 28%|██████████████████████████████▉                                                                               | 702/2499 [1:24:59<12:01:51, 24.10s/it]
                                                                                                                                                           
{'loss': 0.5731, 'grad_norm': 0.16955940425395966, 'learning_rate': 0.00016431635466365588, 'ppl': 1.7738, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4483.16, 'total_tokens': 30225994, 'epoch': 0.84}
-
 28%|██████████████████████████████▉                                                                               | 702/2499 [1:24:59<12:01:51, 24.10s/it]
 28%|███████████████████████████████▏                                                                               | 703/2499 [1:25:05<9:21:18, 18.75s/it]
                                                                                                                                                           
{'loss': 0.5525, 'grad_norm': 0.15249280631542206, 'learning_rate': 0.00016421965387556088, 'ppl': 1.7376, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4607.61, 'total_tokens': 30254865, 'epoch': 0.84}
-
 28%|███████████████████████████████▏                                                                               | 703/2499 [1:25:05<9:21:18, 18.75s/it]
 28%|███████████████████████████████▎                                                                               | 704/2499 [1:25:11<7:28:49, 15.00s/it]
                                                                                                                                                           
{'loss': 0.5425, 'grad_norm': 0.1723303347826004, 'learning_rate': 0.00016412285077751997, 'ppl': 1.7203, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4369.08, 'total_tokens': 30282176, 'epoch': 0.85}
-
 28%|███████████████████████████████▎                                                                               | 704/2499 [1:25:11<7:28:49, 15.00s/it]
 28%|███████████████████████████████▎                                                                               | 705/2499 [1:25:17<6:10:12, 12.38s/it]
                                                                                                                                                           
{'loss': 0.5519, 'grad_norm': 0.15896157920360565, 'learning_rate': 0.00016402594552375263, 'ppl': 1.7365, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4565.22, 'total_tokens': 30310768, 'epoch': 0.85}
-
 28%|███████████████████████████████▎                                                                               | 705/2499 [1:25:17<6:10:12, 12.38s/it]
 28%|███████████████████████████████▎                                                                               | 706/2499 [1:25:24<5:15:39, 10.56s/it]
                                                                                                                                                           
{'loss': 0.5752, 'grad_norm': 0.16958479583263397, 'learning_rate': 0.00016392893826864104, 'ppl': 1.7775, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4643.26, 'total_tokens': 30340100, 'epoch': 0.85}
-
 28%|███████████████████████████████▎                                                                               | 706/2499 [1:25:24<5:15:39, 10.56s/it]
 28%|███████████████████████████████▍                                                                               | 707/2499 [1:25:30<4:37:28,  9.29s/it]
                                                                                                                                                           
{'loss': 0.5338, 'grad_norm': 0.1593128889799118, 'learning_rate': 0.00016383182916672988, 'ppl': 1.7054, 'memory/max_active (GiB)': 17.82, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4479.9, 'total_tokens': 30368404, 'epoch': 0.85}
-
 28%|███████████████████████████████▍                                                                               | 707/2499 [1:25:30<4:37:28,  9.29s/it]
 28%|███████████████████████████████▍                                                                               | 708/2499 [1:25:36<4:10:27,  8.39s/it]
                                                                                                                                                           
{'loss': 0.612, 'grad_norm': 0.17639009654521942, 'learning_rate': 0.0001637346183727261, 'ppl': 1.8441, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4575.21, 'total_tokens': 30397166, 'epoch': 0.85}
-
 28%|███████████████████████████████▍                                                                               | 708/2499 [1:25:36<4:10:27,  8.39s/it]
 28%|███████████████████████████████▍                                                                               | 709/2499 [1:25:43<3:51:15,  7.75s/it]
                                                                                                                                                           
{'loss': 0.5829, 'grad_norm': 0.16138216853141785, 'learning_rate': 0.0001636373060414987, 'ppl': 1.7912, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4527.66, 'total_tokens': 30425501, 'epoch': 0.85}
-
 28%|███████████████████████████████▍                                                                               | 709/2499 [1:25:43<3:51:15,  7.75s/it]
 28%|███████████████████████████████▌                                                                               | 710/2499 [1:25:49<3:38:03,  7.31s/it]
                                                                                                                                                           
{'loss': 0.5956, 'grad_norm': 0.15134470164775848, 'learning_rate': 0.00016353989232807838, 'ppl': 1.8141, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4679.04, 'total_tokens': 30454926, 'epoch': 0.85}
-
 28%|███████████████████████████████▌                                                                               | 710/2499 [1:25:49<3:38:03,  7.31s/it]
 28%|███████████████████████████████▌                                                                               | 711/2499 [1:25:55<3:28:35,  7.00s/it]
                                                                                                                                                           
{'loss': 0.568, 'grad_norm': 0.15280140936374664, 'learning_rate': 0.00016344237738765736, 'ppl': 1.7647, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4557.84, 'total_tokens': 30483483, 'epoch': 0.85}
-
 28%|███████████████████████████████▌                                                                               | 711/2499 [1:25:55<3:28:35,  7.00s/it]
 28%|███████████████████████████████▋                                                                               | 712/2499 [1:26:01<3:21:51,  6.78s/it]
                                                                                                                                                           
{'loss': 0.5629, 'grad_norm': 0.1602308750152588, 'learning_rate': 0.00016334476137558918, 'ppl': 1.7558, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4569.03, 'total_tokens': 30512071, 'epoch': 0.85}
-
 28%|███████████████████████████████▋                                                                               | 712/2499 [1:26:01<3:21:51,  6.78s/it]
 29%|███████████████████████████████▋                                                                               | 713/2499 [1:26:08<3:17:22,  6.63s/it]
                                                                                                                                                           
{'loss': 0.5874, 'grad_norm': 0.1549394726753235, 'learning_rate': 0.00016324704444738833, 'ppl': 1.7993, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4828.45, 'total_tokens': 30542421, 'epoch': 0.86}
-
 29%|███████████████████████████████▋                                                                               | 713/2499 [1:26:08<3:17:22,  6.63s/it]
 29%|███████████████████████████████▋                                                                               | 714/2499 [1:26:14<3:14:10,  6.53s/it]
                                                                                                                                                           
{'loss': 0.5733, 'grad_norm': 0.16239850223064423, 'learning_rate': 0.0001631492267587301, 'ppl': 1.7741, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4661.58, 'total_tokens': 30571708, 'epoch': 0.86}
-
 29%|███████████████████████████████▋                                                                               | 714/2499 [1:26:14<3:14:10,  6.53s/it]
 29%|███████████████████████████████▊                                                                               | 715/2499 [1:26:20<3:11:56,  6.46s/it]
                                                                                                                                                           
{'loss': 0.522, 'grad_norm': 0.154056578874588, 'learning_rate': 0.00016305130846545036, 'ppl': 1.6854, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4365.39, 'total_tokens': 30599150, 'epoch': 0.86}
-
 29%|███████████████████████████████▊                                                                               | 715/2499 [1:26:20<3:11:56,  6.46s/it]
 29%|███████████████████████████████▊                                                                               | 716/2499 [1:26:27<3:10:06,  6.40s/it]
                                                                                                                                                           
{'loss': 0.5257, 'grad_norm': 0.1525743454694748, 'learning_rate': 0.00016295328972354515, 'ppl': 1.6916, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4667.34, 'total_tokens': 30628366, 'epoch': 0.86}
-
 29%|███████████████████████████████▊                                                                               | 716/2499 [1:26:27<3:10:06,  6.40s/it]
 29%|███████████████████████████████▊                                                                               | 717/2499 [1:26:33<3:08:44,  6.36s/it]
                                                                                                                                                           
{'loss': 0.5435, 'grad_norm': 0.1493917852640152, 'learning_rate': 0.00016285517068917064, 'ppl': 1.722, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4464.33, 'total_tokens': 30656287, 'epoch': 0.86}
-
 29%|███████████████████████████████▊                                                                               | 717/2499 [1:26:33<3:08:44,  6.36s/it]
 29%|███████████████████████████████▉                                                                               | 718/2499 [1:26:39<3:07:36,  6.32s/it]
                                                                                                                                                           
{'loss': 0.5321, 'grad_norm': 0.15722903609275818, 'learning_rate': 0.0001627569515186427, 'ppl': 1.7025, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4426.51, 'total_tokens': 30683897, 'epoch': 0.86}
-
 29%|███████████████████████████████▉                                                                               | 718/2499 [1:26:39<3:07:36,  6.32s/it]
 29%|███████████████████████████████▉                                                                               | 719/2499 [1:26:45<3:06:59,  6.30s/it]
                                                                                                                                                           
{'loss': 0.6011, 'grad_norm': 0.15095295011997223, 'learning_rate': 0.00016265863236843676, 'ppl': 1.8241, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4590.16, 'total_tokens': 30712629, 'epoch': 0.86}
-
 29%|███████████████████████████████▉                                                                               | 719/2499 [1:26:45<3:06:59,  6.30s/it]
 29%|███████████████████████████████▉                                                                               | 720/2499 [1:26:52<3:06:24,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5869, 'grad_norm': 0.15645472705364227, 'learning_rate': 0.00016256021339518758, 'ppl': 1.7984, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4481.63, 'total_tokens': 30740626, 'epoch': 0.86}
-
 29%|███████████████████████████████▉                                                                               | 720/2499 [1:26:52<3:06:24,  6.29s/it]
 29%|████████████████████████████████                                                                               | 721/2499 [1:26:58<3:06:28,  6.29s/it]
                                                                                                                                                           
{'loss': 0.4806, 'grad_norm': 0.15596990287303925, 'learning_rate': 0.00016246169475568885, 'ppl': 1.617, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4632.04, 'total_tokens': 30769829, 'epoch': 0.87}
-
 29%|████████████████████████████████                                                                               | 721/2499 [1:26:58<3:06:28,  6.29s/it]
 29%|████████████████████████████████                                                                               | 722/2499 [1:27:04<3:06:12,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5909, 'grad_norm': 0.18050049245357513, 'learning_rate': 0.0001623630766068931, 'ppl': 1.8056, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4588.45, 'total_tokens': 30798606, 'epoch': 0.87}
-
 29%|████████████████████████████████                                                                               | 722/2499 [1:27:04<3:06:12,  6.29s/it]
 29%|████████████████████████████████                                                                               | 723/2499 [1:27:10<3:05:43,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5575, 'grad_norm': 0.15670737624168396, 'learning_rate': 0.00016226435910591137, 'ppl': 1.7463, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4363.67, 'total_tokens': 30825842, 'epoch': 0.87}
-
 29%|████████████████████████████████                                                                               | 723/2499 [1:27:10<3:05:43,  6.27s/it]
 29%|████████████████████████████████▏                                                                              | 724/2499 [1:27:17<3:05:20,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5402, 'grad_norm': 0.1583596020936966, 'learning_rate': 0.00016216554241001304, 'ppl': 1.7164, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4469.72, 'total_tokens': 30853736, 'epoch': 0.87}
-
 29%|████████████████████████████████▏                                                                              | 724/2499 [1:27:17<3:05:20,  6.26s/it]
 29%|████████████████████████████████▏                                                                              | 725/2499 [1:27:23<3:05:10,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5711, 'grad_norm': 0.16624586284160614, 'learning_rate': 0.00016206662667662543, 'ppl': 1.7702, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4475.01, 'total_tokens': 30881736, 'epoch': 0.87}
-
 29%|████████████████████████████████▏                                                                              | 725/2499 [1:27:23<3:05:10,  6.26s/it]
 29%|████████████████████████████████▏                                                                              | 726/2499 [1:27:29<3:05:00,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5675, 'grad_norm': 0.1706494837999344, 'learning_rate': 0.00016196761206333372, 'ppl': 1.7639, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.21, 'tokens_per_second_per_gpu': 4510.05, 'total_tokens': 30909939, 'epoch': 0.87}
-
 29%|████████████████████████████████▏                                                                              | 726/2499 [1:27:29<3:05:00,  6.26s/it]
 29%|████████████████████████████████▎                                                                              | 727/2499 [1:27:35<3:05:09,  6.27s/it]
                                                                                                                                                           
{'loss': 0.4818, 'grad_norm': 0.15161283314228058, 'learning_rate': 0.0001618684987278806, 'ppl': 1.619, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4650.17, 'total_tokens': 30939175, 'epoch': 0.87}
-
 29%|████████████████████████████████▎                                                                              | 727/2499 [1:27:35<3:05:09,  6.27s/it]
 29%|████████████████████████████████▎                                                                              | 728/2499 [1:27:42<3:04:58,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5631, 'grad_norm': 0.15963025391101837, 'learning_rate': 0.00016176928682816593, 'ppl': 1.7561, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4538.66, 'total_tokens': 30967571, 'epoch': 0.87}
-
 29%|████████████████████████████████▎                                                                              | 728/2499 [1:27:42<3:04:58,  6.27s/it]
 29%|████████████████████████████████▍                                                                              | 729/2499 [1:27:48<3:04:38,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5683, 'grad_norm': 0.15409500896930695, 'learning_rate': 0.00016166997652224675, 'ppl': 1.7653, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4435.31, 'total_tokens': 30995244, 'epoch': 0.88}
-
 29%|████████████████████████████████▍                                                                              | 729/2499 [1:27:48<3:04:38,  6.26s/it]
 29%|████████████████████████████████▍                                                                              | 730/2499 [1:27:54<3:04:25,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5345, 'grad_norm': 0.1505703181028366, 'learning_rate': 0.00016157056796833676, 'ppl': 1.7066, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4474.51, 'total_tokens': 31023185, 'epoch': 0.88}
-
 29%|████████████████████████████████▍                                                                              | 730/2499 [1:27:54<3:04:25,  6.26s/it]
 29%|████████████████████████████████▍                                                                              | 731/2499 [1:28:00<3:04:22,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5862, 'grad_norm': 0.15465688705444336, 'learning_rate': 0.00016147106132480626, 'ppl': 1.7971, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4602.88, 'total_tokens': 31051995, 'epoch': 0.88}
-
 29%|████████████████████████████████▍                                                                              | 731/2499 [1:28:00<3:04:22,  6.26s/it]
 29%|████████████████████████████████▌                                                                              | 732/2499 [1:28:07<3:04:03,  6.25s/it]
                                                                                                                                                           
{'loss': 0.5423, 'grad_norm': 0.16041669249534607, 'learning_rate': 0.0001613714567501818, 'ppl': 1.72, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4323.79, 'total_tokens': 31078937, 'epoch': 0.88}
-
 29%|████████████████████████████████▌                                                                              | 732/2499 [1:28:07<3:04:03,  6.25s/it]
 29%|████████████████████████████████▌                                                                              | 733/2499 [1:28:13<3:04:13,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5709, 'grad_norm': 0.15239129960536957, 'learning_rate': 0.00016127175440314596, 'ppl': 1.7699, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4708.34, 'total_tokens': 31108497, 'epoch': 0.88}
-
 29%|████████████████████████���███████▌                                                                              | 733/2499 [1:28:13<3:04:13,  6.26s/it]
 29%|████████████████████████████████▌                                                                              | 734/2499 [1:28:19<3:04:30,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5298, 'grad_norm': 0.15932819247245789, 'learning_rate': 0.00016117195444253696, 'ppl': 1.6986, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4452.06, 'total_tokens': 31136547, 'epoch': 0.88}
-
 29%|████████████████████████████████▌                                                                              | 734/2499 [1:28:19<3:04:30,  6.27s/it]
 29%|████████████████████████████████▋                                                                              | 735/2499 [1:28:25<3:04:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5579, 'grad_norm': 0.16990432143211365, 'learning_rate': 0.0001610720570273487, 'ppl': 1.747, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4584.98, 'total_tokens': 31165311, 'epoch': 0.88}
-
 29%|████████████████████████████████▋                                                                              | 735/2499 [1:28:25<3:04:26,  6.27s/it]
 29%|████████████████████████████████▋                                                                              | 736/2499 [1:28:32<3:04:20,  6.27s/it]
                                                                                                                                                           
{'loss': 0.605, 'grad_norm': 0.1646908074617386, 'learning_rate': 0.00016097206231673023, 'ppl': 1.8313, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4569.2, 'total_tokens': 31193969, 'epoch': 0.88}
-
 29%|████████████████████████████████▋                                                                              | 736/2499 [1:28:32<3:04:20,  6.27s/it]
 29%|████████████████████████████████▋                                                                              | 737/2499 [1:28:38<3:04:00,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5668, 'grad_norm': 0.16785207390785217, 'learning_rate': 0.00016087197046998566, 'ppl': 1.7626, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4384.7, 'total_tokens': 31221351, 'epoch': 0.88}
-
 29%|████████████████████████████████▋                                                                              | 737/2499 [1:28:38<3:04:00,  6.27s/it]
 30%|████████████████████████████████▊                                                                              | 738/2499 [1:28:44<3:03:49,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5376, 'grad_norm': 0.1626354157924652, 'learning_rate': 0.00016077178164657384, 'ppl': 1.7119, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4431.69, 'total_tokens': 31249067, 'epoch': 0.89}
-
 30%|████████████████████████████████▊                                                                              | 738/2499 [1:28:44<3:03:49,  6.26s/it]
 30%|████████████████████████████████▊                                                                              | 739/2499 [1:28:51<3:03:45,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5673, 'grad_norm': 0.155157670378685, 'learning_rate': 0.00016067149600610807, 'ppl': 1.7635, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4556.76, 'total_tokens': 31277622, 'epoch': 0.89}
-
 30%|████████████████████████████████▊                                                                              | 739/2499 [1:28:51<3:03:45,  6.26s/it]
 30%|████████████████████████████████▊                                                                              | 740/2499 [1:28:57<3:04:05,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5287, 'grad_norm': 0.15059494972229004, 'learning_rate': 0.00016057111370835597, 'ppl': 1.6967, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4635.29, 'total_tokens': 31306875, 'epoch': 0.89}
-
 30%|████████████████████████████████▊                                                                              | 740/2499 [1:28:57<3:04:05,  6.28s/it]
 30%|████████████████████████████████▉                                                                              | 741/2499 [1:29:03<3:04:00,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5562, 'grad_norm': 0.17175139486789703, 'learning_rate': 0.00016047063491323905, 'ppl': 1.744, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4406.56, 'total_tokens': 31334542, 'epoch': 0.89}
-
 30%|████████████████████████████████▉                                                                              | 741/2499 [1:29:03<3:04:00,  6.28s/it]
 30%|████████████████████████████████▉                                                                              | 742/2499 [1:29:09<3:03:55,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5384, 'grad_norm': 0.1837666630744934, 'learning_rate': 0.00016037005978083264, 'ppl': 1.7133, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4472.03, 'total_tokens': 31362623, 'epoch': 0.89}
-
 30%|████████████████████████████████▉                                                                              | 742/2499 [1:29:09<3:03:55,  6.28s/it]
 30%|█████████████████████████████████                                                                              | 743/2499 [1:29:16<3:03:44,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5274, 'grad_norm': 0.1696690171957016, 'learning_rate': 0.00016026938847136547, 'ppl': 1.6945, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4592.04, 'total_tokens': 31391413, 'epoch': 0.89}
-
 30%|█████████████████████████████████                                                                              | 743/2499 [1:29:16<3:03:44,  6.28s/it]
 30%|█████████████████████████████████                                                                              | 744/2499 [1:29:22<3:03:33,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5377, 'grad_norm': 0.16248169541358948, 'learning_rate': 0.00016016862114521963, 'ppl': 1.7121, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4493.63, 'total_tokens': 31419578, 'epoch': 0.89}
-
 30%|█████████████████████████████████                                                                              | 744/2499 [1:29:22<3:03:33,  6.28s/it]
 30%|█████████████████████████████████                                                                              | 745/2499 [1:29:28<3:03:21,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5036, 'grad_norm': 0.1554591804742813, 'learning_rate': 0.00016006775796293003, 'ppl': 1.6547, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4533.36, 'total_tokens': 31447965, 'epoch': 0.89}
-
 30%|█████████████████████████████████                                                                              | 745/2499 [1:29:28<3:03:21,  6.27s/it]
 30%|█████████████████████████████████▏                                                                             | 746/2499 [1:29:35<3:03:28,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5041, 'grad_norm': 0.16030484437942505, 'learning_rate': 0.0001599667990851843, 'ppl': 1.6555, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4585.24, 'total_tokens': 31476832, 'epoch': 0.9}
-
 30%|█████████████████████████████████▏                                                                             | 746/2499 [1:29:35<3:03:28,  6.28s/it]
 30%|█████████████████████████████████▏                                                                             | 747/2499 [1:29:41<3:03:35,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5446, 'grad_norm': 0.16970005631446838, 'learning_rate': 0.0001598657446728226, 'ppl': 1.7239, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4677.17, 'total_tokens': 31506305, 'epoch': 0.9}
-
 30%|█████████████████████████████████▏                                                                             | 747/2499 [1:29:41<3:03:35,  6.29s/it]
 30%|█████████████████████████████████▏                                                                             | 748/2499 [1:29:47<3:03:18,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5235, 'grad_norm': 0.15128681063652039, 'learning_rate': 0.00015976459488683728, 'ppl': 1.6879, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4639.67, 'total_tokens': 31535371, 'epoch': 0.9}
-
 30%|█████████████████████████████████▏                                                                             | 748/2499 [1:29:47<3:03:18,  6.28s/it]
 30%|█████████████████████████████████▎                                                                             | 749/2499 [1:29:53<3:03:01,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5248, 'grad_norm': 0.1509389579296112, 'learning_rate': 0.00015966334988837258, 'ppl': 1.6901, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4623.7, 'total_tokens': 31564315, 'epoch': 0.9}
-
 30%|█████████████████████████████████▎                                                                             | 749/2499 [1:29:53<3:03:01,  6.28s/it]
 30%|█████████████████████████████████▎                                                                             | 750/2499 [1:30:00<3:02:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5745, 'grad_norm': 0.16577394306659698, 'learning_rate': 0.00015956200983872448, 'ppl': 1.7762, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4551.42, 'total_tokens': 31592819, 'epoch': 0.9}
-
 30%|█████████████████████████████████▎                                                                             | 750/2499 [1:30:00<3:02:50,  6.27s/it]
 30%|█████████████████████████████████▎                                                                             | 751/2499 [1:30:06<3:02:39,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5912, 'grad_norm': 0.15780609846115112, 'learning_rate': 0.00015946057489934034, 'ppl': 1.8062, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4580.38, 'total_tokens': 31621502, 'epoch': 0.9}
-
 30%|█████████████████████████████████▎                                                                             | 751/2499 [1:30:06<3:02:39,  6.27s/it]
 30%|█████████████████████████████████▍                                                                             | 752/2499 [1:30:12<3:02:52,  6.28s/it]
                                                                                                                                                           
{'loss': 0.4995, 'grad_norm': 0.15355201065540314, 'learning_rate': 0.0001593590452318187, 'ppl': 1.6479, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4607.08, 'total_tokens': 31650544, 'epoch': 0.9}
-
 30%|█████████████████████████████████▍                                                                             | 752/2499 [1:30:12<3:02:52,  6.28s/it]
 30%|█████████████████████████████████▍                                                                             | 753/2499 [1:30:18<3:02:41,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5469, 'grad_norm': 0.16567420959472656, 'learning_rate': 0.00015925742099790906, 'ppl': 1.7279, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4397.59, 'total_tokens': 31678110, 'epoch': 0.9}
-
 30%|█████████████████████████████████▍                                                                             | 753/2499 [1:30:18<3:02:41,  6.28s/it]
 30%|█████████████████████████████████▍                                                                             | 754/2499 [1:30:25<3:02:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5663, 'grad_norm': 0.1523253619670868, 'learning_rate': 0.00015915570235951152, 'ppl': 1.7617, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4396.31, 'total_tokens': 31705623, 'epoch': 0.91}
-
 30%|█████████████████████████████████▍                                                                             | 754/2499 [1:30:25<3:02:26,  6.27s/it]
 30%|█████████████████████████████████▌                                                                             | 755/2499 [1:30:31<3:02:03,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5856, 'grad_norm': 0.16661697626113892, 'learning_rate': 0.00015905388947867662, 'ppl': 1.7961, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4343.58, 'total_tokens': 31732726, 'epoch': 0.91}
-
 30%|█████████████████████████████████▌                                                                             | 755/2499 [1:30:31<3:02:03,  6.26s/it]
 30%|█████████████████████████████████▌                                                                             | 756/2499 [1:30:37<3:01:56,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5887, 'grad_norm': 0.16167092323303223, 'learning_rate': 0.00015895198251760498, 'ppl': 1.8016, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4538.97, 'total_tokens': 31761137, 'epoch': 0.91}
-
 30%|█████████████████████████████████▌                                                                             | 756/2499 [1:30:37<3:01:56,  6.26s/it]
 30%|█████████████████████████████████▌                                                                             | 757/2499 [1:30:43<3:01:51,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5713, 'grad_norm': 0.17005272209644318, 'learning_rate': 0.0001588499816386472, 'ppl': 1.7706, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4576.99, 'total_tokens': 31789807, 'epoch': 0.91}
-
 30%|█████████████████████████████████▌                                                                             | 757/2499 [1:30:43<3:01:51,  6.26s/it]
 30%|█████████████████████████████████▋                                                                             | 758/2499 [1:30:50<3:01:53,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5854, 'grad_norm': 0.175898477435112, 'learning_rate': 0.0001587478870043034, 'ppl': 1.7957, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4658.87, 'total_tokens': 31819052, 'epoch': 0.91}
-
 30%|█████████████████████████████████▋                                                                             | 758/2499 [1:30:50<3:01:53,  6.27s/it]
 30%|█████████████████████████████████▋                                                                             | 759/2499 [1:30:56<3:02:07,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5608, 'grad_norm': 0.15077874064445496, 'learning_rate': 0.00015864569877722316, 'ppl': 1.7521, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4518.24, 'total_tokens': 31847537, 'epoch': 0.91}
-
 30%|█████████████████████████████████▋                                                                             | 759/2499 [1:30:56<3:02:07,  6.28s/it]
 30%|█████████████████████████████████▊                                                                             | 760/2499 [1:31:02<3:01:40,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5751, 'grad_norm': 0.1493740826845169, 'learning_rate': 0.00015854341712020508, 'ppl': 1.7773, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4319.39, 'total_tokens': 31874476, 'epoch': 0.91}
-
 30%|█████████████████████████████████▊                                                                             | 760/2499 [1:31:02<3:01:40,  6.27s/it]
 30%|█████████████████████████████████▊                                                                             | 761/2499 [1:31:09<3:01:26,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5335, 'grad_norm': 0.1708398163318634, 'learning_rate': 0.00015844104219619665, 'ppl': 1.7049, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4383.94, 'total_tokens': 31901879, 'epoch': 0.91}
-
 30%|█████████████████████████████████▊                                                                             | 761/2499 [1:31:09<3:01:26,  6.26s/it]
 30%|█████████████████████████████████▊                                                                             | 762/2499 [1:31:15<3:01:16,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5301, 'grad_norm': 0.14762543141841888, 'learning_rate': 0.00015833857416829388, 'ppl': 1.6991, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4475.92, 'total_tokens': 31929878, 'epoch': 0.91}
-
 30%|█████████████████████████████████▊                                                                             | 762/2499 [1:31:15<3:01:16,  6.26s/it]
 31%|█████████████████████████████████▉                                                                             | 763/2499 [1:31:21<3:01:17,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6072, 'grad_norm': 0.1617388278245926, 'learning_rate': 0.00015823601319974126, 'ppl': 1.8353, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4742.64, 'total_tokens': 31959631, 'epoch': 0.92}
-
 31%|█████████████████████████████████▉                                                                             | 763/2499 [1:31:21<3:01:17,  6.27s/it]
 31%|█████████████████████████████████▉                                                                             | 764/2499 [1:31:27<3:01:13,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5234, 'grad_norm': 0.1600262075662613, 'learning_rate': 0.0001581333594539312, 'ppl': 1.6878, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4627.93, 'total_tokens': 31988635, 'epoch': 0.92}
-
 31%|█████████████████████████████████▉                                                                             | 764/2499 [1:31:27<3:01:13,  6.27s/it]
 31%|█████████████████████████████████▉                                                                             | 765/2499 [1:31:34<3:01:31,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5283, 'grad_norm': 0.14879342913627625, 'learning_rate': 0.00015803061309440394, 'ppl': 1.696, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4565.48, 'total_tokens': 32017455, 'epoch': 0.92}
-
 31%|█████████████████████████████████▉                                                                             | 765/2499 [1:31:34<3:01:31,  6.28s/it]
 31%|██████████████████████████████████                                                                             | 766/2499 [1:31:40<3:01:13,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5884, 'grad_norm': 0.15819264948368073, 'learning_rate': 0.00015792777428484737, 'ppl': 1.8011, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4405.56, 'total_tokens': 32045015, 'epoch': 0.92}
-
 31%|██████████████████████████████████                                                                             | 766/2499 [1:31:40<3:01:13,  6.27s/it]
 31%|██████████████████████████████████                                                                             | 767/2499 [1:31:46<3:00:57,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5787, 'grad_norm': 0.1562289297580719, 'learning_rate': 0.00015782484318909646, 'ppl': 1.7837, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4493.41, 'total_tokens': 32073112, 'epoch': 0.92}
-
 31%|██████████████████████████████████                                                                             | 767/2499 [1:31:46<3:00:57,  6.27s/it]
 31%|██████████████████████████████████                                                                             | 768/2499 [1:31:52<3:00:50,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5137, 'grad_norm': 0.15492472052574158, 'learning_rate': 0.00015772181997113344, 'ppl': 1.6715, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4498.38, 'total_tokens': 32101299, 'epoch': 0.92}
-
 31%|██████████████████████████████████                                                                             | 768/2499 [1:31:52<3:00:50,  6.27s/it]
 31%|██████████████████████████████████▏                                                                            | 769/2499 [1:31:59<3:00:39,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5724, 'grad_norm': 0.15212927758693695, 'learning_rate': 0.00015761870479508717, 'ppl': 1.7725, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4483.39, 'total_tokens': 32129344, 'epoch': 0.92}
-
 31%|██████████████████████████████████▏                                                                            | 769/2499 [1:31:59<3:00:39,  6.27s/it]
 31%|██████████████████████████████████▏                                                                            | 770/2499 [1:32:05<3:00:31,  6.26s/it]
                                                                                                                                                           
{'loss': 0.554, 'grad_norm': 0.1595274955034256, 'learning_rate': 0.000157515497825233, 'ppl': 1.7402, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4437.13, 'total_tokens': 32157127, 'epoch': 0.92}
-
 31%|██████████████████████████████████▏                                                                            | 770/2499 [1:32:05<3:00:31,  6.26s/it]
 31%|██████████████████████████████████▏                                                                            | 771/2499 [1:32:11<3:00:39,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5321, 'grad_norm': 0.15527474880218506, 'learning_rate': 0.00015741219922599253, 'ppl': 1.7025, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4421.95, 'total_tokens': 32184942, 'epoch': 0.93}
-
 31%|██████████████████████████████████▏                                                                            | 771/2499 [1:32:11<3:00:39,  6.27s/it]
 31%|██████████████████████████████████▎                                                                            | 772/2499 [1:32:18<3:00:53,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5571, 'grad_norm': 0.15651631355285645, 'learning_rate': 0.00015730880916193339, 'ppl': 1.7456, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4396.43, 'total_tokens': 32212676, 'epoch': 0.93}
-
 31%|██████████████████████████████████▎                                                                            | 772/2499 [1:32:18<3:00:53,  6.28s/it]
 31%|██████████████████████████████████▎                                                                            | 773/2499 [1:32:24<3:00:47,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5452, 'grad_norm': 0.16266778111457825, 'learning_rate': 0.0001572053277977689, 'ppl': 1.725, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4635.82, 'total_tokens': 32241789, 'epoch': 0.93}
-
 31%|██████████████████████████████████▎                                                                            | 773/2499 [1:32:24<3:00:47,  6.28s/it]
 31%|██████████████████████████████████▍                                                                            | 774/2499 [1:32:30<3:00:34,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5728, 'grad_norm': 0.16812606155872345, 'learning_rate': 0.0001571017552983578, 'ppl': 1.7732, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4453.43, 'total_tokens': 32269716, 'epoch': 0.93}
-
 31%|██████████████████████████████████▍                                                                            | 774/2499 [1:32:30<3:00:34,  6.28s/it]
 31%|██████████████████████████████████▍                                                                            | 775/2499 [1:32:36<3:00:18,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5525, 'grad_norm': 0.17196150124073029, 'learning_rate': 0.00015699809182870408, 'ppl': 1.7376, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4301.26, 'total_tokens': 32296637, 'epoch': 0.93}
-
 31%|██████████████████████████████████▍                                                                            | 775/2499 [1:32:36<3:00:18,  6.28s/it]
 31%|██████████████████████████████████▍                                                                            | 776/2499 [1:32:43<3:00:11,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5695, 'grad_norm': 0.15856219828128815, 'learning_rate': 0.00015689433755395658, 'ppl': 1.7674, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4565.26, 'total_tokens': 32325266, 'epoch': 0.93}
-
 31%|██████████████████████████████████▍                                                                            | 776/2499 [1:32:43<3:00:11,  6.27s/it]
 31%|██████████████████████████████████▌                                                                            | 777/2499 [1:32:49<2:59:54,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5745, 'grad_norm': 0.16656816005706787, 'learning_rate': 0.00015679049263940886, 'ppl': 1.7762, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4389.85, 'total_tokens': 32352709, 'epoch': 0.93}
-
 31%|██████████████████████████████████▌                                                                            | 777/2499 [1:32:49<2:59:54,  6.27s/it]
 31%|██████████████████████████████████▌                                                                            | 778/2499 [1:32:55<3:00:02,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5508, 'grad_norm': 0.16310793161392212, 'learning_rate': 0.00015668655725049885, 'ppl': 1.7346, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4532.43, 'total_tokens': 32381234, 'epoch': 0.93}
-
 31%|██████████████████████████████████▌                                                                            | 778/2499 [1:32:55<3:00:02,  6.28s/it]
 31%|██████████████████████████████████▌                                                                            | 779/2499 [1:33:02<2:59:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5621, 'grad_norm': 0.16509102284908295, 'learning_rate': 0.00015658253155280869, 'ppl': 1.7544, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4355.28, 'total_tokens': 32408569, 'epoch': 0.94}
-
 31%|██████████████████████████████████▌                                                                            | 779/2499 [1:33:02<2:59:57,  6.28s/it]
 31%|██████████████████████████████████▋                                                                            | 780/2499 [1:33:08<2:59:46,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5407, 'grad_norm': 0.15348419547080994, 'learning_rate': 0.00015647841571206426, 'ppl': 1.7172, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4583.34, 'total_tokens': 32437286, 'epoch': 0.94}
-
 31%|██████████████████████████████████▋                                                                            | 780/2499 [1:33:08<2:59:46,  6.27s/it]
 31%|██████████████████████████████████▋                                                                            | 781/2499 [1:33:14<2:59:37,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5852, 'grad_norm': 0.1582723706960678, 'learning_rate': 0.00015637420989413517, 'ppl': 1.7954, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4560.78, 'total_tokens': 32465872, 'epoch': 0.94}
-
 31%|██████████████████████████████████▋                                                                            | 781/2499 [1:33:14<2:59:37,  6.27s/it]
 31%|██████████████████████████████████▋                                                                            | 782/2499 [1:33:20<2:59:22,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5516, 'grad_norm': 0.15862534940242767, 'learning_rate': 0.00015626991426503434, 'ppl': 1.736, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4551.21, 'total_tokens': 32494335, 'epoch': 0.94}
-
 31%|██████████████████████████████████▋                                                                            | 782/2499 [1:33:20<2:59:22,  6.27s/it]
 31%|██████████████████████████████████▊                                                                            | 783/2499 [1:33:27<2:59:23,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5604, 'grad_norm': 0.17056556046009064, 'learning_rate': 0.0001561655289909177, 'ppl': 1.7514, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4619.88, 'total_tokens': 32523348, 'epoch': 0.94}
-
 31%|██████████████████████████████████▊                                                                            | 783/2499 [1:33:27<2:59:23,  6.27s/it]
 31%|██████████████████████████████████▊                                                                            | 784/2499 [1:33:33<2:59:09,  6.27s/it]
                                                                                                                                                           
{'loss': 0.522, 'grad_norm': 0.1711031049489975, 'learning_rate': 0.0001560610542380841, 'ppl': 1.6854, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4453.59, 'total_tokens': 32551204, 'epoch': 0.94}
-
 31%|██████████████████████████████████▊                                                                            | 784/2499 [1:33:33<2:59:09,  6.27s/it]
 31%|██████████████████████████████████▊                                                                            | 785/2499 [1:33:39<2:59:11,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5556, 'grad_norm': 0.15930446982383728, 'learning_rate': 0.00015595649017297495, 'ppl': 1.743, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4334.13, 'total_tokens': 32578431, 'epoch': 0.94}
-
 31%|██████████████████████████████████▊                                                                            | 785/2499 [1:33:39<2:59:11,  6.27s/it]
 31%|██████████████████████████████████▉                                                                            | 786/2499 [1:33:45<2:59:06,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5105, 'grad_norm': 0.15259447693824768, 'learning_rate': 0.00015585183696217376, 'ppl': 1.6661, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4524.64, 'total_tokens': 32606804, 'epoch': 0.94}
-
 31%|██████████████████████████████████▉                                                                            | 786/2499 [1:33:45<2:59:06,  6.27s/it]
 31%|██████████████████████████████████▉                                                                            | 787/2499 [1:33:52<2:58:56,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5947, 'grad_norm': 0.16778172552585602, 'learning_rate': 0.0001557470947724063, 'ppl': 1.8125, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4500.07, 'total_tokens': 32634993, 'epoch': 0.94}
-
 31%|██████████████████████████████████▉                                                                            | 787/2499 [1:33:52<2:58:56,  6.27s/it]
 32%|███████████████████████████████████                                                                            | 788/2499 [1:33:58<2:58:56,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5268, 'grad_norm': 0.1585581749677658, 'learning_rate': 0.00015564226377053997, 'ppl': 1.6935, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4699.12, 'total_tokens': 32664513, 'epoch': 0.95}
-
 32%|███████████████████████████████████                                                                            | 788/2499 [1:33:58<2:58:56,  6.28s/it]
 32%|███████████████████████████████████                                                                            | 789/2499 [1:34:04<2:58:46,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5573, 'grad_norm': 0.16107720136642456, 'learning_rate': 0.0001555373441235836, 'ppl': 1.746, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4517.19, 'total_tokens': 32692816, 'epoch': 0.95}
-
 32%|███████████████████████████████████                                                                            | 789/2499 [1:34:04<2:58:46,  6.27s/it]
 32%|███████████████████████████████████                                                                            | 790/2499 [1:34:10<2:58:30,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5212, 'grad_norm': 0.16135258972644806, 'learning_rate': 0.00015543233599868742, 'ppl': 1.684, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4423.93, 'total_tokens': 32720470, 'epoch': 0.95}
-
 32%|███████████████████████████████████                                                                            | 790/2499 [1:34:10<2:58:30,  6.27s/it]
 32%|███████████████████████████████████▏                                                                           | 791/2499 [1:34:17<2:58:21,  6.27s/it]
                                                                                                                                                           
{'loss': 0.5447, 'grad_norm': 0.163282573223114, 'learning_rate': 0.0001553272395631425, 'ppl': 1.7241, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4540.97, 'total_tokens': 32748900, 'epoch': 0.95}
-
 32%|███████████████████████████████████▏                                                                           | 791/2499 [1:34:17<2:58:21,  6.27s/it]
 32%|███████████████████████████████████▏                                                                           | 792/2499 [1:34:23<2:58:26,  6.27s/it]
                                                                                                                                                           
{'loss': 0.4688, 'grad_norm': 0.16475090384483337, 'learning_rate': 0.00015522205498438055, 'ppl': 1.5981, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4425.91, 'total_tokens': 32776712, 'epoch': 0.95}
-
 32%|███████████████████████████████████▏                                                                           | 792/2499 [1:34:23<2:58:26,  6.27s/it]
 32%|███████████████████████████████████▏                                                                           | 793/2499 [1:34:29<2:58:20,  6.27s/it]
                                                                                                                                                           
{'loss': 0.6043, 'grad_norm': 0.15547741949558258, 'learning_rate': 0.0001551167824299738, 'ppl': 1.83, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4378.39, 'total_tokens': 32804162, 'epoch': 0.95}
-
 32%|███████████████████████████████████▏                                                                           | 793/2499 [1:34:29<2:58:20,  6.27s/it]
 32%|███████████████████████████████████▎                                                                           | 794/2499 [1:34:36<2:57:59,  6.26s/it]
                                                                                                                                                           
{'loss': 0.572, 'grad_norm': 0.17574763298034668, 'learning_rate': 0.0001550114220676346, 'ppl': 1.7718, 'memory/max_active (GiB)': 18.08, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4278.44, 'total_tokens': 32830864, 'epoch': 0.95}
-
 32%|███████████████████████████████████▎                                                                           | 794/2499 [1:34:36<2:57:59,  6.26s/it]
 32%|███████████████████████████████████▎                                                                           | 795/2499 [1:34:42<2:57:47,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5346, 'grad_norm': 0.1534729152917862, 'learning_rate': 0.00015490597406521523, 'ppl': 1.7068, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4439.31, 'total_tokens': 32858614, 'epoch': 0.95}
-
 32%|███████████████████████████████████▎                                                                           | 795/2499 [1:34:42<2:57:47,  6.26s/it]
 32%|███████████████████████████████████▎                                                                           | 796/2499 [1:34:48<2:57:28,  6.25s/it]
                                                                                                                                                           
{'loss': 0.5156, 'grad_norm': 0.15551310777664185, 'learning_rate': 0.0001548004385907075, 'ppl': 1.6746, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4302.36, 'total_tokens': 32885428, 'epoch': 0.96}
-
 32%|███████████████████████████████████▎                                                                           | 796/2499 [1:34:48<2:57:28,  6.25s/it]
 32%|███████████████████████████████████▍                                                                           | 797/2499 [1:34:54<2:57:33,  6.26s/it]
                                                                                                                                                           
{'loss': 0.5561, 'grad_norm': 0.1516779214143753, 'learning_rate': 0.00015469481581224272, 'ppl': 1.7439, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4665.25, 'total_tokens': 32914695, 'epoch': 0.96}
-
 32%|███████████████████████████████████▍                                                                           | 797/2499 [1:34:54<2:57:33,  6.26s/it]
 32%|███████████████████████████████████▍                                                                           | 798/2499 [1:35:01<2:57:57,  6.28s/it]
                                                                                                                                                           
{'loss': 0.6199, 'grad_norm': 0.15012866258621216, 'learning_rate': 0.00015458910589809106, 'ppl': 1.8587, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4722.7, 'total_tokens': 32944520, 'epoch': 0.96}
-
 32%|███████████████████████████████████▍                                                                           | 798/2499 [1:35:01<2:57:57,  6.28s/it]
 32%|███████████████████████████████████▍                                                                           | 799/2499 [1:35:07<2:58:13,  6.29s/it]
                                                                                                                                                           
{'loss': 0.5909, 'grad_norm': 0.16060224175453186, 'learning_rate': 0.00015448330901666175, 'ppl': 1.8056, 'memory/max_active (GiB)': 18.05, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4528.95, 'total_tokens': 32973133, 'epoch': 0.96}
-
 32%|███████████████████████████████████▍                                                                           | 799/2499 [1:35:07<2:58:13,  6.29s/it]
 32%|███████████████████████████████████▌                                                                           | 800/2499 [1:35:13<2:57:56,  6.28s/it]
                                                                                                                                                           
{'loss': 0.5605, 'grad_norm': 0.16525596380233765, 'learning_rate': 0.00015437742533650243, 'ppl': 1.7515, 'memory/max_active (GiB)': 18.11, 'memory/max_allocated (GiB)': 12.52, 'memory/device_reserved (GiB)': 20.25, 'tokens_per_second_per_gpu': 4510.81, 'total_tokens': 33001395, 'epoch': 0.96}
-
 32%|███████████████████████████████████▌                                                                           | 800/2499 [1:35:13<2:57:56,  6.28s/it][2025-12-28 12:40:49,853] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:42410] Running evaluation step...
-[2025-12-28 12:40:51,617] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8908905982971191
-[2025-12-28 12:40:52,485] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.867694616317749
-[2025-12-28 12:40:53,341] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8542790412902832
-[2025-12-28 12:40:54,200] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:42410] generate_batches time: 0.8589270114898682
-[2025-12-28 12:40:54,201] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:42410] gather_len_batches: [90]
+
  0%|                                                                                                                               | 0/72 [00:00<?, ?it/s]
+
  3%|███▎                                                                                                                   | 2/72 [00:01<01:08,  1.03it/s]
+
  4%|████▉                                                                                                                  | 3/72 [00:03<01:34,  1.37s/it]
+
  6%|██████▌                                                                                                                | 4/72 [00:05<01:47,  1.58s/it]
+
  7%|████████▎                                                                                                              | 5/72 [00:07<01:53,  1.69s/it]
+
  8%|█████████▉                                                                                                             | 6/72 [00:09<01:57,  1.77s/it]
+
 10%|███████████▌                                                                                                           | 7/72 [00:11<02:00,  1.85s/it]
+
 11%|█████████████▏                                                                                                         | 8/72 [00:13<02:00,  1.89s/it]
+
 12%|██████████████▉                                                                                                        | 9/72 [00:15<01:59,  1.90s/it]
+
 14%|████████████████▍                                                                                                     | 10/72 [00:17<01:57,  1.90s/it]
+
 15%|██████████████████                                                                                                    | 11/72 [00:19<01:56,  1.91s/it]
+
 17%|███████████████████▋                                                                                                  | 12/72 [00:21<01:54,  1.92s/it]
+
 18%|█████████████████████▎                                                                                                | 13/72 [00:23<01:53,  1.92s/it]
+
 19%|██████████████████████▉                                                                                               | 14/72 [00:25<01:51,  1.92s/it]
+
 21%|████████████████████████▌                                                                                             | 15/72 [00:27<01:49,  1.93s/it]
+
 22%|██████████████████████████▏                                                                                           | 16/72 [00:29<01:47,  1.93s/it]
+
 24%|███████████████████████████▊                                                                                          | 17/72 [00:30<01:46,  1.93s/it]
+
 25%|█████████████████████████████▌                                                                                        | 18/72 [00:32<01:44,  1.93s/it]
+
 26%|███████████████████████████████▏                                                                                      | 19/72 [00:34<01:41,  1.92s/it]
+
 28%|████████████████████████████████▊                                                                                     | 20/72 [00:36<01:39,  1.91s/it]
+
 29%|██████████████████████████████████▍                                                                                   | 21/72 [00:38<01:37,  1.92s/it]
+
 31%|████████████████████████████████████                                                                                  | 22/72 [00:40<01:36,  1.92s/it]
+
 32%|█████████████████████████████████████▋                                                                                | 23/72 [00:42<01:34,  1.92s/it]
+
 33%|███████████████████████████████████████▎                                                                              | 24/72 [00:45<01:41,  2.12s/it]
+
 35%|████████████████████████████████████████▉                                                                             | 25/72 [00:47<01:43,  2.21s/it]
+
 36%|████████████��█████████████████████████████▌                                                                           | 26/72 [00:49<01:40,  2.19s/it]
+
 38%|████████████████████████████████████████████▎                                                                         | 27/72 [00:51<01:35,  2.11s/it]
+
 39%|█████████████████████████████████████████████▉                                                                        | 28/72 [00:53<01:30,  2.05s/it]
+
 40%|███████████████████████████████████████████████▌                                                                      | 29/72 [00:55<01:32,  2.16s/it]
+
 42%|█████████████████████████████████████████████████▏                                                                    | 30/72 [00:57<01:27,  2.08s/it]
+
 43%|██████████████████████████████████████████████████▊                                                                   | 31/72 [00:59<01:23,  2.03s/it]
+
 44%|████████████████████████████████████████████████████▍                                                                 | 32/72 [01:01<01:19,  2.00s/it]
+
 46%|██████████████████████████████████████████████████████                                                                | 33/72 [01:03<01:16,  1.97s/it]
+
 47%|███████████████████████████████████████████████████████▋                                                              | 34/72 [01:05<01:14,  1.96s/it]
+
 49%|█████████████████████████████████████████████████████████▎                                                            | 35/72 [01:07<01:12,  1.95s/it]
+
 50%|███████████████████████████████████████████████████████████                                                           | 36/72 [01:09<01:09,  1.94s/it]
+
 51%|████████████████████████████████████████████████████████████▋                                                         | 37/72 [01:11<01:07,  1.94s/it]
+
 53%|██████████████████████████████████████████████████████████████▎                                                       | 38/72 [01:13<01:05,  1.93s/it]
+
 54%|███████████████████████████████████████████████████████████████▉                                                      | 39/72 [01:15<01:03,  1.92s/it]
+
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 40/72 [01:16<01:01,  1.91s/it]
+
 57%|███████████████████████████████████████████████████████████████████▏                                                  | 41/72 [01:18<00:59,  1.92s/it]
+
 58%|████████████████████████████████████████████████████████████████████▊                                                 | 42/72 [01:20<00:57,  1.92s/it]
+
 60%|██████████████████████████████████████████████████████████████████████▍                                               | 43/72 [01:22<00:55,  1.92s/it]
+
 61%|████████████████████████████████████████████████████████████████████████                                              | 44/72 [01:24<00:53,  1.92s/it]
+
 62%|█████████████████████████████████████████████████████████████████████████▊                                            | 45/72 [01:26<00:51,  1.92s/it]
+
 64%|███████████████████████████████████████████████████████████████████████████▍                                          | 46/72 [01:28<00:49,  1.92s/it]
+
 65%|█████████████████████████████████████████████████████████████████████████████                                         | 47/72 [01:30<00:48,  1.92s/it]
+
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 48/72 [01:32<00:46,  1.93s/it]
+
 68%|████████████████████████████████████████████████████████████████████████████████▎                                     | 49/72 [01:34<00:44,  1.92s/it]
+
 69%|█████████████████████████████████████████████████████████████████████████████████▉                                    | 50/72 [01:36<00:42,  1.91s/it]
+
 71%|███████████████████████████████████████████████████████████████████████████████████▌                                  | 51/72 [01:38<00:40,  1.91s/it]
+
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 52/72 [01:39<00:38,  1.91s/it]
+
 74%|██████████████████████████████████████████████████████████████████████████████████████▊                               | 53/72 [01:41<00:36,  1.91s/it]
+
 75%|████████████████████████████████████████████████████████████████████████████████████████▌                             | 54/72 [01:43<00:34,  1.91s/it]
+
 76%|██████████████████████████████████████████████████████████████████████████████████████████▏                           | 55/72 [01:45<00:32,  1.92s/it]
+
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 56/72 [01:47<00:30,  1.92s/it]
+
 79%|█████████████████████████████████████████████████████████████████████████████████████████████▍                        | 57/72 [01:49<00:28,  1.92s/it]
+
 81%|███████████████████████████████████████████████████████████████████████████████████████████████                       | 58/72 [01:51<00:26,  1.92s/it]
+
 82%|████████████████████████████████████████████████████████████████████████████████████████████████▋                     | 59/72 [01:53<00:25,  1.92s/it]
+
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 60/72 [01:55<00:22,  1.92s/it]
+
 85%|███████████████████████████████████████████████████████████████████████████████████████████████████▉                  | 61/72 [01:57<00:21,  1.91s/it]
+
 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌                | 62/72 [01:59<00:19,  1.91s/it]
+
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎              | 63/72 [02:01<00:17,  1.91s/it]
+
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 64/72 [02:02<00:15,  1.92s/it]
+
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 65/72 [02:04<00:13,  1.92s/it]
+
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 66/72 [02:06<00:11,  1.92s/it]
+
 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 67/72 [02:08<00:09,  1.92s/it]
+
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 68/72 [02:10<00:07,  1.92s/it]
+
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████     | 69/72 [02:13<00:06,  2.07s/it]
+
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 70/72 [02:15<00:04,  2.02s/it]
+
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 71/72 [02:16<00:01,  1.98s/it]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:18<00:00,  1.98s/it]
                                                                                                                                                           
+
                                                                                                                                                           
{'eval_loss': 0.5063687562942505, 'eval_runtime': 141.112, 'eval_samples_per_second': 5.173, 'eval_steps_per_second': 1.035, 'eval_ppl': 1.6593, 'memory/max_active (GiB)': 19.1, 'memory/max_allocated (GiB)': 19.1, 'memory/device_reserved (GiB)': 139.06, 'epoch': 1.5}
+
 50%|███████████████████████████████████████████████████████                                                       | 2000/3996 [2:30:47<2:21:13,  4.25s/it]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:19<00:00,  1.98s/it]
+
                                                                                                                                                           [2025-12-29 05:20:32,819] [INFO] [axolotl.core.trainers.base._save:692] [PID:3751] Saving model checkpoint to ./outputs/luau-codellama-h200-fast/checkpoint-2000
+
 50%|██████████████████████████████████████████████████████▌                                                      | 2001/3996 [2:30:51<26:11:28, 47.26s/it]
 50%|██████████████████████████████████████████████████████▌                                                      | 2002/3996 [2:30:56<19:01:20, 34.34s/it]
 50%|██████████████████████████████████████████████████████▋                                                      | 2003/3996 [2:31:00<14:05:39, 25.46s/it]
 50%|██████████████████████████████████████████████████████▋                                                      | 2004/3996 [2:31:05<10:33:08, 19.07s/it]
 50%|███████████████████████████████████████████████████████▏                                                      | 2005/3996 [2:31:09<8:04:36, 14.60s/it]
 50%|███████████████████████████████████████████████████████▏                                                      | 2006/3996 [2:31:13<6:20:41, 11.48s/it]
 50%|███████████████████████████████████████████████████████▏                                                      | 2007/3996 [2:31:17<5:08:09,  9.30s/it]
 50%|███████████████████████████████████████████████████████▎                                                      | 2008/3996 [2:31:21<4:17:15,  7.76s/it]
 50%|███████████████████████████████████████████████████████▎                                                      | 2009/3996 [2:31:25<3:41:44,  6.70s/it]
 50%|███████████████████████████████████████████████████████▎                                                      | 2010/3996 [2:31:30<3:22:07,  6.11s/it]
 50%|███████████████████████████████████████████████████████▎                                                      | 2011/3996 [2:31:34<3:02:58,  5.53s/it]
 50%|███████████████████████████████████████████████████████▍                                                      | 2012/3996 [2:31:39<2:49:31,  5.13s/it]
 50%|███████████████████████████████████████████████████████▍                                                      | 2013/3996 [2:31:43<2:40:02,  4.84s/it]
 50%|███████████████████████████████████████████████████████▍                                                      | 2014/3996 [2:31:47<2:33:31,  4.65s/it]
 50%|███████████████████████████████████████████████████████▍                                                      | 2015/3996 [2:31:51<2:28:54,  4.51s/it]
 50%|███████████████████████████████████████████████████████▍                                                      | 2016/3996 [2:31:55<2:25:42,  4.42s/it]
 50%|███████████████████████████████████████████████████████▌                                                      | 2017/3996 [2:32:00<2:28:50,  4.51s/it]
 51%|███████████████████████████████████████████████████████▌                                                      | 2018/3996 [2:32:04<2:25:25,  4.41s/it]
 51%|███████████████████████████████████████████████████████▌                                                      | 2019/3996 [2:32:08<2:23:09,  4.34s/it]
 51%|███████████████████████████████████████████████████████▌                                                      | 2020/3996 [2:32:13<2:21:32,  4.30s/it]
 51%|███████████████████████████████████████████████████████▋                                                      | 2021/3996 [2:32:17<2:20:37,  4.27s/it]
 51%|███████████████████████████████████████████████████████▋                                                      | 2022/3996 [2:32:21<2:19:42,  4.25s/it]
 51%|███████████████████████████████████████████████████████▋                                                      | 2023/3996 [2:32:25<2:19:11,  4.23s/it]
 51%|███████████████████████████████████████████████████████▋                                                      | 2024/3996 [2:32:30<2:24:06,  4.38s/it]
 51%|███████████████████████████████████████████████████████▋                                                      | 2025/3996 [2:32:34<2:22:09,  4.33s/it]
                                                                                                                                                           
{'loss': 0.4892, 'grad_norm': 0.20089760422706604, 'learning_rate': 0.00010193515188183245, 'ppl': 1.631, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4246.58, 'total_tokens': 39959888, 'epoch': 1.52}
+
 51%|███████████████████████████████████████████████████████▋                                                      | 2025/3996 [2:32:34<2:22:09,  4.33s/it]
 51%|███████████████████████████████████████████████████████▊                                                      | 2026/3996 [2:32:38<2:20:50,  4.29s/it]
 51%|███████████████████████████████████████████████████████▊                                                      | 2027/3996 [2:32:43<2:19:48,  4.26s/it]
 51%|███████████████████████████████████████████████████████▊                                                      | 2028/3996 [2:32:47<2:19:10,  4.24s/it]
 51%|███████████████████████████████████████████████████████▊                                                      | 2029/3996 [2:32:51<2:18:36,  4.23s/it]
 51%|███████████████████████████████████████████████████████▉                                                      | 2030/3996 [2:32:55<2:18:18,  4.22s/it]
 51%|█████████████████████████████████████████████████████��█▉                                                      | 2031/3996 [2:33:00<2:23:17,  4.38s/it]
 51%|███████████████████████████████████████████████████████▉                                                      | 2032/3996 [2:33:04<2:21:34,  4.32s/it]
 51%|███████████████████████████████████████████████████████▉                                                      | 2033/3996 [2:33:08<2:20:03,  4.28s/it]
 51%|███████████████████████████████████████████████████████▉                                                      | 2034/3996 [2:33:12<2:19:02,  4.25s/it]
 51%|████████████████████████████████████████████████████████                                                      | 2035/3996 [2:33:17<2:18:28,  4.24s/it]
 51%|████████████████████████████████████████████████████████                                                      | 2036/3996 [2:33:21<2:18:01,  4.23s/it]
 51%|████████████████████████████████████████████████████████                                                      | 2037/3996 [2:33:25<2:17:37,  4.22s/it]
 51%|████████████████████████████████████████████████████████                                                      | 2038/3996 [2:33:30<2:22:35,  4.37s/it]
 51%|████████████████████████████████████████████████████████▏                                                     | 2039/3996 [2:33:34<2:20:50,  4.32s/it]
 51%|████████████████████████████████████████████████████████▏                                                     | 2040/3996 [2:33:38<2:19:34,  4.28s/it]
 51%|████████████████████████████████████████████████████████▏                                                     | 2041/3996 [2:33:42<2:18:29,  4.25s/it]
 51%|████████████████████████████████████████████████████████▏                                                     | 2042/3996 [2:33:47<2:17:48,  4.23s/it]
 51%|████████████████████████████████████████████████████████▏                                                     | 2043/3996 [2:33:51<2:17:29,  4.22s/it]
 51%|████████████████████████████████████████████████████████▎                                                     | 2044/3996 [2:33:55<2:17:15,  4.22s/it]
 51%|████████████████████████████████████████████████████████▎                                                     | 2045/3996 [2:34:00<2:22:13,  4.37s/it]
 51%|████████████████████████████████████████████████████████▎                                                     | 2046/3996 [2:34:04<2:20:20,  4.32s/it]
 51%|████████████████████████████████████████████████████████▎                                                     | 2047/3996 [2:34:08<2:18:58,  4.28s/it]
 51%|████████████████████████████████████████████████████████▍                                                     | 2048/3996 [2:34:12<2:18:00,  4.25s/it]
 51%|████████████████████████████████████████████████████████▍                                                     | 2049/3996 [2:34:16<2:17:29,  4.24s/it]
 51%|███████████████████████████████████████████████████████��▍                                                     | 2050/3996 [2:34:21<2:16:55,  4.22s/it]
                                                                                                                                                           
{'loss': 0.503, 'grad_norm': 0.19840118288993835, 'learning_rate': 9.991936364699348e-05, 'ppl': 1.6537, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4320.38, 'total_tokens': 40411902, 'epoch': 1.54}
+
 51%|████████████████████████████████████████████████████████▍                                                     | 2050/3996 [2:34:21<2:16:55,  4.22s/it]
 51%|████████████████████████████████████████████████████████▍                                                     | 2051/3996 [2:34:25<2:16:36,  4.21s/it]
 51%|████████████████████████████████████████████████████████▍                                                     | 2052/3996 [2:34:30<2:21:38,  4.37s/it]
 51%|████████████████████████████████████████████████████████▌                                                     | 2053/3996 [2:34:34<2:19:57,  4.32s/it]
 51%|████████████████████████████████████████████████████████▌                                                     | 2054/3996 [2:34:38<2:18:41,  4.29s/it]
 51%|████████████████████████████████████████████████████████▌                                                     | 2055/3996 [2:34:42<2:17:36,  4.25s/it]
 51%|████████████████████████████████████████████████████████▌                                                     | 2056/3996 [2:34:46<2:17:03,  4.24s/it]
 51%|████████████████████████████████████████████████████████▌                                                     | 2057/3996 [2:34:51<2:16:38,  4.23s/it]
 52%|████████████████████████████████████████████████████████▋                                                     | 2058/3996 [2:34:55<2:16:21,  4.22s/it]
 52%|████████████████████████████████████████████████████████▋                                                     | 2059/3996 [2:35:00<2:21:17,  4.38s/it]
 52%|████████████████████████████████████████████████████████▋                                                     | 2060/3996 [2:35:04<2:19:31,  4.32s/it]
 52%|████████████████████████████████████████████████████████▋                                                     | 2061/3996 [2:35:08<2:18:08,  4.28s/it]
 52%|████████████████████████████████████████████████████████▊                                                     | 2062/3996 [2:35:12<2:17:04,  4.25s/it]
 52%|████████████████████████████████████████████████████████▊                                                     | 2063/3996 [2:35:16<2:16:27,  4.24s/it]
 52%|████████████████████████████████████████████████████████▊                                                     | 2064/3996 [2:35:20<2:16:06,  4.23s/it]
 52%|████████████████████████████████████████████████████████▊                                                     | 2065/3996 [2:35:25<2:15:44,  4.22s/it]
 52%|████████████████████████████████████████████████████████▊                                                     | 2066/3996 [2:35:29<2:20:38,  4.37s/it]
 52%|████████████████████████████████████████████████████████▉                                                     | 2067/3996 [2:35:34<2:18:50,  4.32s/it]
 52%|████████████████████████████████████████████████████████▉                                                     | 2068/3996 [2:35:38<2:17:37,  4.28s/it]
 52%|████████████████████████████████████████████████████████▉                                                     | 2069/3996 [2:35:42<2:16:27,  4.25s/it]
 52%|████████████████████████████████████████████████████████▉                                                     | 2070/3996 [2:35:46<2:15:53,  4.23s/it]
 52%|█████████████████████████████████████████████████████████                                                     | 2071/3996 [2:35:50<2:15:21,  4.22s/it]
 52%|█████████████████████████████████████████████████████████                                                     | 2072/3996 [2:35:55<2:15:05,  4.21s/it]
 52%|█████████████████████████████████████████████████████████                                                     | 2073/3996 [2:35:59<2:20:33,  4.39s/it]
 52%|█████████████████████████████████████████████████████████                                                     | 2074/3996 [2:36:04<2:18:36,  4.33s/it]
 52%|█████████████████████████████████████████████████████████                                                     | 2075/3996 [2:36:08<2:17:07,  4.28s/it]
                                                                                                                                                           
{'loss': 0.5127, 'grad_norm': 0.20045842230319977, 'learning_rate': 9.790360818076577e-05, 'ppl': 1.6698, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4245.02, 'total_tokens': 40855384, 'epoch': 1.56}
+
 52%|█████████████████████████████████████████████████████████                                                     | 2075/3996 [2:36:08<2:17:07,  4.28s/it]
 52%|█████████████████████████████████████████████████████████▏                                                    | 2076/3996 [2:36:12<2:16:11,  4.26s/it]
 52%|█████████████████████████████████████████████████████████▏                                                    | 2077/3996 [2:36:16<2:15:35,  4.24s/it]
 52%|█████████████████████████████████████████████████████████▏                                                    | 2078/3996 [2:36:20<2:15:04,  4.23s/it]
 52%|█████████████████████████████████████████████████████████▏                                                    | 2079/3996 [2:36:24<2:14:43,  4.22s/it]
 52%|█████████████████████████████████████████████████████████▎                                                    | 2080/3996 [2:36:29<2:19:32,  4.37s/it]
 52%|█████████████████████████████████████████████████████████▎                                                    | 2081/3996 [2:36:33<2:17:43,  4.32s/it]
 52%|█████████████████████████████████████████████████████████▎                                                    | 2082/3996 [2:36:38<2:16:26,  4.28s/it]
 52%|█████████████████████████████████████████████████████████▎                                                    | 2083/3996 [2:36:42<2:15:22,  4.25s/it]
 52%|█████████████████████████████████████████████████████████▎                                                    | 2084/3996 [2:36:46<2:14:56,  4.23s/it]
 52%|█████████████████████████████████████████████████████████▍                                                    | 2085/3996 [2:36:50<2:14:19,  4.22s/it]
 52%|█████████████████████████████████████████████████████████▍                                                    | 2086/3996 [2:36:54<2:14:02,  4.21s/it]
 52%|█████████████████████████████████████████████████████████▍                                                    | 2087/3996 [2:36:59<2:18:55,  4.37s/it]
 52%|█████████████████████████████████████████████████████████▍                                                    | 2088/3996 [2:37:03<2:17:08,  4.31s/it]
 52%|█████████████████████████████████████████████████████████▌                                                    | 2089/3996 [2:37:07<2:15:46,  4.27s/it]
 52%|█████████████████████████████████████████████████████████▌                                                    | 2090/3996 [2:37:12<2:14:50,  4.24s/it]
 52%|█████████████████████████████████████████████████████████▌                                                    | 2091/3996 [2:37:16<2:14:14,  4.23s/it]
 52%|█████████████████████████████████████████████████████████▌                                                    | 2092/3996 [2:37:20<2:13:37,  4.21s/it]
 52%|█████████████████████████████████████████████████████████▌                                                    | 2093/3996 [2:37:24<2:13:22,  4.21s/it]
 52%|█████████████████████████████████████████████████████████▋                                                    | 2094/3996 [2:37:29<2:18:18,  4.36s/it]
 52%|█████████████████████████████████████████████████████████▋                                                    | 2095/3996 [2:37:33<2:16:42,  4.32s/it]
 52%|█████████████████████████████████████████████████████████▋                                                    | 2096/3996 [2:37:37<2:15:22,  4.27s/it]
 52%|█████████████████████████████████████████████████████████▋                                                    | 2097/3996 [2:37:41<2:14:21,  4.25s/it]
 53%|█████████████████████████████████████████████████████████▊                                                    | 2098/3996 [2:37:46<2:13:42,  4.23s/it]
 53%|█████████████████████████████████████████████████████████▊                                                    | 2099/3996 [2:37:50<2:13:22,  4.22s/it]
 53%|█████████████████████████████████████████████████████████▊                                                    | 2100/3996 [2:37:54<2:13:07,  4.21s/it]
                                                                                                                                                           
{'loss': 0.4994, 'grad_norm': 0.19669026136398315, 'learning_rate': 9.588870463610893e-05, 'ppl': 1.6477, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4174.18, 'total_tokens': 41293525, 'epoch': 1.58}
+
 53%|█████████████████████████████████████████████████████████▊                                                    | 2100/3996 [2:37:54<2:13:07,  4.21s/it]
 53%|█████████████████████████████████████████████████████████▊                                                    | 2101/3996 [2:37:59<2:17:58,  4.37s/it]
 53%|█████████████████████████████████████████████████████████▊                                                    | 2102/3996 [2:38:03<2:16:10,  4.31s/it]
 53%|█████████████████████████████████████████████████████████▉                                                    | 2103/3996 [2:38:07<2:14:52,  4.27s/it]
 53%|█████████████████████████████████████████████████████████▉                                                    | 2104/3996 [2:38:11<2:13:55,  4.25s/it]
 53%|█████████████████████████████████████████████████████████▉                                                    | 2105/3996 [2:38:16<2:13:31,  4.24s/it]
 53%|█████████████████████████████████████████████████████████▉                                                    | 2106/3996 [2:38:20<2:12:58,  4.22s/it]
 53%|██████████████████████████████████████████████████████████                                                    | 2107/3996 [2:38:24<2:13:01,  4.23s/it]
 53%|██████████████████████████████████████████████████████████                                                    | 2108/3996 [2:38:29<2:17:50,  4.38s/it]
 53%|██████████████████████████████████████████████████████████                                                    | 2109/3996 [2:38:33<2:16:04,  4.33s/it]
 53%|██████████████████████████████████████████████████████████                                                    | 2110/3996 [2:38:37<2:14:47,  4.29s/it]
 53%|██████████████████████████████████████████████████████████                                                    | 2111/3996 [2:38:41<2:13:41,  4.26s/it]
 53%|██████████████████████████████████████████████████████████▏                                                   | 2112/3996 [2:38:45<2:13:05,  4.24s/it]
 53%|██████████████████████████████████████████████████████████▏                                                   | 2113/3996 [2:38:50<2:12:37,  4.23s/it]
 53%|██████████████████████████████████████████████████████████▏                                                   | 2114/3996 [2:38:54<2:12:11,  4.21s/it]
 53%|██████████████████████████████████████████████████████████▏                                                   | 2115/3996 [2:38:59<2:16:58,  4.37s/it]
 53%|██████████████████████████████████████████████████████████▏                                                   | 2116/3996 [2:39:03<2:16:40,  4.36s/it]
 53%|██████████████████████████████████████████████████████████▎                                                   | 2117/3996 [2:39:07<2:18:18,  4.42s/it]
 53%|██████████████████████████████████████████████████████████▎                                                   | 2118/3996 [2:39:12<2:16:11,  4.35s/it]
 53%|██████████████████████████████████████████████████████████▎                                                   | 2119/3996 [2:39:16<2:14:50,  4.31s/it]
 53%|██████████████████████████████████████████████████████████▎                                                   | 2120/3996 [2:39:20<2:13:40,  4.28s/it]
 53%|██████████████████████████████████████████████████████████▍                                                   | 2121/3996 [2:39:24<2:12:49,  4.25s/it]
 53%|██████████████████████████████████████████████████████████▍                                                   | 2122/3996 [2:39:29<2:17:24,  4.40s/it]
 53%|██████████████████████████████████████████████████████████▍                                                   | 2123/3996 [2:39:33<2:15:26,  4.34s/it]
 53%|██████████████████████████████████████████████████████████▍                                                   | 2124/3996 [2:39:37<2:13:57,  4.29s/it]
 53%|██████████████████████████████████████████████████████████▍                                                   | 2125/3996 [2:39:42<2:13:18,  4.27s/it]
                                                                                                                                                           
{'loss': 0.5009, 'grad_norm': 0.19754259288311005, 'learning_rate': 9.387547181978291e-05, 'ppl': 1.6502, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4200.06, 'total_tokens': 41737747, 'epoch': 1.59}
+
 53%|██████████████████████████████████████████████████████████▍                                                   | 2125/3996 [2:39:42<2:13:18,  4.27s/it]
 53%|██████████████████████████████████████████████████████████▌                                                   | 2126/3996 [2:39:46<2:12:27,  4.25s/it]
 53%|██████████████████████████████████████████████████████████▌                                                   | 2127/3996 [2:39:50<2:11:49,  4.23s/it]
 53%|██████████████████████████████████████████████████████████▌                                                   | 2128/3996 [2:39:54<2:11:24,  4.22s/it]
 53%|██████████████████████████████████████████████████████████▌                                                   | 2129/3996 [2:39:59<2:16:14,  4.38s/it]
 53%|██████████████████████████████████████████████████████████▋                                                   | 2130/3996 [2:40:03<2:14:32,  4.33s/it]
 53%|██████████████████████████████████████████████████████████▋                                                   | 2131/3996 [2:40:07<2:13:12,  4.29s/it]
 53%|██████████████████████████████████████████████████████████▋                                                   | 2132/3996 [2:40:12<2:12:10,  4.25s/it]
 53%|██████████████████████████████████████████████████████████▋                                                   | 2133/3996 [2:40:16<2:11:34,  4.24s/it]
 53%|████████��█████████████████████████████████████████████████▋                                                   | 2134/3996 [2:40:20<2:10:59,  4.22s/it]
 53%|██████████████████████████████████████████████████████████▊                                                   | 2135/3996 [2:40:24<2:10:40,  4.21s/it]
 53%|██████████████████████████████████████████████████████████▊                                                   | 2136/3996 [2:40:29<2:15:35,  4.37s/it]
 53%|██████████████████████████████████████████████████████████▊                                                   | 2137/3996 [2:40:33<2:13:48,  4.32s/it]
 54%|██████████████████████████████████████████████████████████▊                                                   | 2138/3996 [2:40:37<2:12:38,  4.28s/it]
 54%|██████████████████████████████████████████████████████████▉                                                   | 2139/3996 [2:40:41<2:11:39,  4.25s/it]
 54%|██████████████████████████████████████████████████████████▉                                                   | 2140/3996 [2:40:46<2:11:04,  4.24s/it]
 54%|██████████████████████████████████████████████████████████▉                                                   | 2141/3996 [2:40:50<2:10:29,  4.22s/it]
 54%|██████████████████████████████████████████████████████████▉                                                   | 2142/3996 [2:40:54<2:10:12,  4.21s/it]
 54%|██████████████████████████████████████████████████████████▉                                                   | 2143/3996 [2:40:59<2:19:14,  4.51s/it]
 54%|███████████████████████████████████████████████████████████                                                   | 2144/3996 [2:41:03<2:16:19,  4.42s/it]
 54%|███████████████████████████████████████████████████████████                                                   | 2145/3996 [2:41:08<2:14:11,  4.35s/it]
 54%|███████████████████████████████████████████████████████████                                                   | 2146/3996 [2:41:12<2:12:37,  4.30s/it]
 54%|███████████████████████████████████████████████████████████                                                   | 2147/3996 [2:41:16<2:11:45,  4.28s/it]
 54%|███████████████████████████████████████████████████████████▏                                                  | 2148/3996 [2:41:20<2:10:55,  4.25s/it]
 54%|███████████████████████████████████████████████████████████▏                                                  | 2149/3996 [2:41:24<2:10:20,  4.23s/it]
 54%|███████████████████████████████████████████████████████████▏                                                  | 2150/3996 [2:41:29<2:14:58,  4.39s/it]
                                                                                                                                                           
{'loss': 0.5002, 'grad_norm': 0.19482502341270447, 'learning_rate': 9.186472785960507e-05, 'ppl': 1.6491, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3696.76, 'total_tokens': 42176082, 'epoch': 1.61}
+
 54%|█████████���█████████████████████████████████████████████████▏                                                  | 2150/3996 [2:41:29<2:14:58,  4.39s/it]
 54%|███████████████████████████████████████████████████████████▏                                                  | 2151/3996 [2:41:33<2:13:08,  4.33s/it]
 54%|███████████████████████████████████████████████████████████▏                                                  | 2152/3996 [2:41:38<2:11:51,  4.29s/it]
 54%|███████████████████████████████████████████████████████████▎                                                  | 2153/3996 [2:41:42<2:10:43,  4.26s/it]
 54%|███████████████████████████████████████████████████████████▎                                                  | 2154/3996 [2:41:46<2:10:11,  4.24s/it]
 54%|███████████████████████████████████████████████████████████▎                                                  | 2155/3996 [2:41:50<2:09:48,  4.23s/it]
 54%|███████████████████████████████████████████████████████████▎                                                  | 2156/3996 [2:41:54<2:09:30,  4.22s/it]
 54%|███████████████████████████████████████████████████████████▍                                                  | 2157/3996 [2:41:59<2:14:21,  4.38s/it]
 54%|███████████████████████████████████████████████████████████▍                                                  | 2158/3996 [2:42:03<2:12:38,  4.33s/it]
 54%|███████████████████████████████████████████████████████████▍                                                  | 2159/3996 [2:42:07<2:11:12,  4.29s/it]
 54%|███████████████████████████████████████████████████████████▍                                                  | 2160/3996 [2:42:12<2:10:09,  4.25s/it]
 54%|███████████████████████████████████████████████████████████▍                                                  | 2161/3996 [2:42:16<2:09:42,  4.24s/it]
 54%|███████████████████████████████████████████████████████████▌                                                  | 2162/3996 [2:42:20<2:09:16,  4.23s/it]
 54%|███████████████████████████████████████████████████████████▌                                                  | 2163/3996 [2:42:24<2:09:01,  4.22s/it]
 54%|███████████████████████████████████████████████████████████▌                                                  | 2164/3996 [2:42:29<2:13:41,  4.38s/it]
 54%|███████████████████████████████████████████████████████████▌                                                  | 2165/3996 [2:42:33<2:11:55,  4.32s/it]
 54%|███████████████████████████████████████████████████████████▌                                                  | 2166/3996 [2:42:37<2:10:36,  4.28s/it]
 54%|███████████████████████████████████████████████████████████▋                                                  | 2167/3996 [2:42:42<2:09:41,  4.25s/it]
 54%|███████████████████████████████████████████████████████████▋                                                  | 2168/3996 [2:42:46<2:08:59,  4.23s/it]
 54%|███████████████████████████████████████████████████████████▋                                                  | 2169/3996 [2:42:50<2:08:24,  4.22s/it]
 54%|███████████████████████████████████████████████████████████▋                                                  | 2170/3996 [2:42:54<2:08:14,  4.21s/it]
 54%|███████████████████████████████████████████████████████████▊                                                  | 2171/3996 [2:42:59<2:13:01,  4.37s/it]
 54%|███████████████████████████████████████████████████████████▊                                                  | 2172/3996 [2:43:03<2:11:18,  4.32s/it]
 54%|███████████████████████████████████████████████████████████▊                                                  | 2173/3996 [2:43:07<2:09:57,  4.28s/it]
 54%|███████████████████████████████████████████████████████████▊                                                  | 2174/3996 [2:43:11<2:08:59,  4.25s/it]
 54%|███████████████████████████████████████████████████████████▊                                                  | 2175/3996 [2:43:16<2:08:29,  4.23s/it]
                                                                                                                                                           
{'loss': 0.4959, 'grad_norm': 0.21606561541557312, 'learning_rate': 8.985728987198352e-05, 'ppl': 1.642, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4192.5, 'total_tokens': 42616372, 'epoch': 1.63}
+
 54%|███████████████████████████████████████████████████████████▊                                                  | 2175/3996 [2:43:16<2:08:29,  4.23s/it]
 54%|███████████████████████████████████████████████████████████▉                                                  | 2176/3996 [2:43:20<2:08:06,  4.22s/it]
 54%|███████████████████████████████████████████████████████████▉                                                  | 2177/3996 [2:43:24<2:07:53,  4.22s/it]
 55%|███████████████████████████████████████████████████████████▉                                                  | 2178/3996 [2:43:29<2:12:30,  4.37s/it]
 55%|███████████████████████████████████████████████████████████▉                                                  | 2179/3996 [2:43:33<2:10:53,  4.32s/it]
 55%|████████████████████████████████████████████████████████████                                                  | 2180/3996 [2:43:37<2:09:41,  4.29s/it]
 55%|████████████████████████████████████████████████████████████                                                  | 2181/3996 [2:43:41<2:08:43,  4.26s/it]
 55%|████████████████████████████████████████████████████████████                                                  | 2182/3996 [2:43:46<2:08:03,  4.24s/it]
 55%|████████████████████████████████████████████████████████████                                                  | 2183/3996 [2:43:50<2:07:31,  4.22s/it]
 55%|████████████████████████████████████████████████████████████                                                  | 2184/3996 [2:43:54<2:07:12,  4.21s/it]
 55%|████████████████████████████████████████████████████████████▏                                                 | 2185/3996 [2:43:59<2:12:03,  4.38s/it]
 55%|████████████████████████████████████████████████████████████▏                                                 | 2186/3996 [2:44:03<2:10:24,  4.32s/it]
 55%|████████████████████████████████████████████████████████████▏                                                 | 2187/3996 [2:44:07<2:09:07,  4.28s/it]
 55%|████████████████████████████████████████████████████████████▏                                                 | 2188/3996 [2:44:11<2:08:05,  4.25s/it]
 55%|████████████████████████████████████████████████████████████▎                                                 | 2189/3996 [2:44:15<2:07:35,  4.24s/it]
 55%|████████████████████████████████████████████████████████████▎                                                 | 2190/3996 [2:44:20<2:07:02,  4.22s/it]
 55%|████████████████████████████████████████████████████████████▎                                                 | 2191/3996 [2:44:24<2:06:47,  4.21s/it]
 55%|████████████████████████████████████████████████████████████▎                                                 | 2192/3996 [2:44:29<2:11:26,  4.37s/it]
 55%|████████████████████████████████████████████████████████████▎                                                 | 2193/3996 [2:44:33<2:09:53,  4.32s/it]
 55%|████████████████████████████████████████████████████████████▍                                                 | 2194/3996 [2:44:37<2:08:44,  4.29s/it]
 55%|████████████████████████████████████████████████████████████▍                                                 | 2195/3996 [2:44:41<2:07:47,  4.26s/it]
 55%|████████████████████████████████████████████████████████████▍                                                 | 2196/3996 [2:44:45<2:07:08,  4.24s/it]
 55%|████████████████████████████████████████████████████████████▍                                                 | 2197/3996 [2:44:50<2:06:34,  4.22s/it]
 55%|████████████████████████████████████████████████████████████▌                                                 | 2198/3996 [2:44:54<2:10:31,  4.36s/it]
 55%|████████████████████████████████████████████████████████████▌                                                 | 2199/3996 [2:44:59<2:13:56,  4.47s/it]
 55%|████████████████████████████████████████████████████████████▌                                                 | 2200/3996 [2:45:03<2:11:24,  4.39s/it]
                                                                                                                                                           
{'loss': 0.5031, 'grad_norm': 0.1979638934135437, 'learning_rate': 8.785397362986114e-05, 'ppl': 1.6538, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4211.67, 'total_tokens': 43058315, 'epoch': 1.65}
+
 55%|████████████████████████████████████████████████████████████▌                                                 | 2200/3996 [2:45:03<2:11:24,  4.39s/it]
 55%|████████████████████████████████████████████████████████████▌                                                 | 2201/3996 [2:45:07<2:09:21,  4.32s/it]
 55%|████████████████████████████████████████████████████████████▌                                                 | 2202/3996 [2:45:12<2:08:05,  4.28s/it]
 55%|████████████████████████████████████████████████████████████▋                                                 | 2203/3996 [2:45:16<2:07:19,  4.26s/it]
 55%|████████████████████████████████████████████████████████████▋                                                 | 2204/3996 [2:45:20<2:06:36,  4.24s/it]
 55%|████████████████████████████████████████████████████████████▋                                                 | 2205/3996 [2:45:24<2:06:16,  4.23s/it]
 55%|████████████████████████████████████████████████████████████▋                                                 | 2206/3996 [2:45:29<2:10:45,  4.38s/it]
 55%|████████████████████████████████████████████████████████████▊                                                 | 2207/3996 [2:45:33<2:09:03,  4.33s/it]
 55%|████████████████████████████████████████████████████████████▊                                                 | 2208/3996 [2:45:37<2:07:49,  4.29s/it]
 55%|████████████████████████████████████████████████████████████▊                                                 | 2209/3996 [2:45:41<2:06:48,  4.26s/it]
 55%|████████████████████████████████████████████████████████████▊                                                 | 2210/3996 [2:45:46<2:06:13,  4.24s/it]
 55%|████████████████████████████████████████████████████████████▊                                                 | 2211/3996 [2:45:50<2:05:38,  4.22s/it]
 55%|████████████████████████████████████████████████████████████▉                                                 | 2212/3996 [2:45:54<2:05:19,  4.22s/it]
 55%|████████████████████████████████████████████████████████████▉                                                 | 2213/3996 [2:45:59<2:09:58,  4.37s/it]
 55%|████████████████████████████████████████████████████████████▉                                                 | 2214/3996 [2:46:03<2:08:17,  4.32s/it]
 55%|████████████████████████████████████████████████████████████▉                                                 | 2215/3996 [2:46:07<2:07:04,  4.28s/it]
 55%|█████████████████████████████████████████████████████████████                                                 | 2216/3996 [2:46:11<2:06:11,  4.25s/it]
 55%|█████████████████████████████████████████████████████████████                                                 | 2217/3996 [2:46:16<2:05:46,  4.24s/it]
 56%|█████████████████████████████████████████████████████████████                                                 | 2218/3996 [2:46:20<2:05:18,  4.23s/it]
 56%|█████████████████████████████████████████████████████████████                                                 | 2219/3996 [2:46:24<2:05:03,  4.22s/it]
 56%|█████████████████████████████████████████████████████████████                                                 | 2220/3996 [2:46:29<2:09:33,  4.38s/it]
 56%|█████████████████████████████████████████████████████████████▏                                                | 2221/3996 [2:46:33<2:07:56,  4.32s/it]
 56%|█████████████████████████████████████████████████████████████▏                                                | 2222/3996 [2:46:37<2:06:35,  4.28s/it]
 56%|█████████████████████████████████████████████████████████████▏                                                | 2223/3996 [2:46:41<2:05:35,  4.25s/it]
 56%|█████████████████████████████████████████████████████████████▏                                                | 2224/3996 [2:46:46<2:05:03,  4.23s/it]
 56%|█████████████████████████████████████████████████████████████▏                                                | 2225/3996 [2:46:50<2:04:32,  4.22s/it]
                                                                                                                                                           
{'loss': 0.4986, 'grad_norm': 0.20717743039131165, 'learning_rate': 8.58555932312059e-05, 'ppl': 1.6464, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4242.04, 'total_tokens': 43501960, 'epoch': 1.67}
+
 56%|█████████████████████████████████████████████████████████████▏                                                | 2225/3996 [2:46:50<2:04:32,  4.22s/it]
 56%|█████████████████████████████████████████████████████████████▎                                                | 2226/3996 [2:46:54<2:04:17,  4.21s/it]
 56%|█████████████████████████████████████████████████████████████▎                                                | 2227/3996 [2:46:59<2:08:49,  4.37s/it]
 56%|█████████████████████████████████████████████████████████████▎                                                | 2228/3996 [2:47:03<2:07:13,  4.32s/it]
 56%|█████████████████████████████████████████████████████████████▎                                                | 2229/3996 [2:47:07<2:05:53,  4.27s/it]
 56%|█████████████████████████████████████████████████████████████▍                                                | 2230/3996 [2:47:11<2:04:58,  4.25s/it]
 56%|█████████████████████████████████████████████████████████████▍                                                | 2231/3996 [2:47:15<2:04:36,  4.24s/it]
 56%|█████████████████████████████████████████████████████████████▍                                                | 2232/3996 [2:47:20<2:04:13,  4.23s/it]
 56%|█████████████████████████████████████████████████████████████▍                                                | 2233/3996 [2:47:24<2:03:57,  4.22s/it]
 56%|██████████████████████████████████████████��██████████████████▍                                                | 2234/3996 [2:47:29<2:08:29,  4.38s/it]
 56%|█████████████████████████████████████████████████████████████▌                                                | 2235/3996 [2:47:33<2:06:51,  4.32s/it]
 56%|█████████████████████████████████████████████████████████████▌                                                | 2236/3996 [2:47:37<2:05:36,  4.28s/it]
 56%|█████████████████████████████████████████████████████████████▌                                                | 2237/3996 [2:47:41<2:04:47,  4.26s/it]
 56%|█████████████████████████████████████████████████████████████▌                                                | 2238/3996 [2:47:45<2:04:12,  4.24s/it]
 56%|█████████████████████████████████████████████████████████████▋                                                | 2239/3996 [2:47:50<2:03:42,  4.22s/it]
 56%|█████████████████████████████████████████████████████████████▋                                                | 2240/3996 [2:47:54<2:03:21,  4.21s/it]
 56%|█████████████████████████████████████████████████████████████▋                                                | 2241/3996 [2:47:58<2:07:58,  4.38s/it]
 56%|█████████████████████████████████████████████████████████████▋                                                | 2242/3996 [2:48:03<2:06:26,  4.33s/it]
 56%|█████████████████████████████████████████████████████████████▋                                                | 2243/3996 [2:48:07<2:05:10,  4.28s/it]
 56%|█████████████████████████████████████████████████████████████▊                                                | 2244/3996 [2:48:11<2:04:23,  4.26s/it]
 56%|█████████████████████████████████████████████████████████████▊                                                | 2245/3996 [2:48:15<2:03:48,  4.24s/it]
 56%|█████████████████████████████████████████████████████████████▊                                                | 2246/3996 [2:48:19<2:03:21,  4.23s/it]
 56%|█████████████████████████████████████████████████████████████▊                                                | 2247/3996 [2:48:24<2:03:04,  4.22s/it]
 56%|█████████████████████████████████████████████████████████████▉                                                | 2248/3996 [2:48:28<2:07:32,  4.38s/it]
 56%|█████████████████████████████████████████████████████████████▉                                                | 2249/3996 [2:48:33<2:06:01,  4.33s/it]
 56%|█████████████████████████████████████████████████████████████▉                                                | 2250/3996 [2:48:37<2:04:53,  4.29s/it]
                                                                                                                                                           
{'loss': 0.4898, 'grad_norm': 0.18736609816551208, 'learning_rate': 8.38629607681815e-05, 'ppl': 1.632, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4235.21, 'total_tokens': 43947235, 'epoch': 1.69}
+
 56%|█████████████████████████████████████████████████████████████▉                                                | 2250/3996 [2:48:37<2:04:53,  4.29s/it]
 56%|█████████████████████████████████████████████████████████████▉                                                | 2251/3996 [2:48:41<2:03:58,  4.26s/it]
 56%|█████████████████████████████████████████████████████████████▉                                                | 2252/3996 [2:48:45<2:03:20,  4.24s/it]
 56%|██████████████████████████████████████████████████████████████                                                | 2253/3996 [2:48:49<2:02:46,  4.23s/it]
 56%|██████████████████████████████████████████████████████████████                                                | 2254/3996 [2:48:54<2:02:35,  4.22s/it]
 56%|██████████████████████████████████████████████████████████████                                                | 2255/3996 [2:48:59<2:10:13,  4.49s/it]
 56%|██████████████████████████████████████████████████████████████                                                | 2256/3996 [2:49:03<2:07:47,  4.41s/it]
 56%|██████████████████████████████████████████████████████████████▏                                               | 2257/3996 [2:49:07<2:05:45,  4.34s/it]
 57%|██████████████████████████████████████████████████████████████▏                                               | 2258/3996 [2:49:11<2:04:30,  4.30s/it]
 57%|██████████████████████████████████████████████████████████████▏                                               | 2259/3996 [2:49:16<2:04:38,  4.31s/it]
 57%|██████████████████████████████████████████████████████████████▏                                               | 2260/3996 [2:49:20<2:03:34,  4.27s/it]
 57%|██████████████████████████████████████████████████████████████▏                                               | 2261/3996 [2:49:24<2:02:46,  4.25s/it]
 57%|██████████████████████████████████████████████████████████████▎                                               | 2262/3996 [2:49:29<2:07:02,  4.40s/it]
 57%|██████████████████████████████████████████████████████████████▎                                               | 2263/3996 [2:49:33<2:05:12,  4.34s/it]
 57%|██████████████████████████████████████████████████████████████▎                                               | 2264/3996 [2:49:37<2:03:57,  4.29s/it]
 57%|██████████████████████████████████████████████████████████████▎                                               | 2265/3996 [2:49:41<2:02:56,  4.26s/it]
 57%|██████████████████████████████████████████████████████████████▍                                               | 2266/3996 [2:49:46<2:02:22,  4.24s/it]
 57%|██████████████████████████████████████████████████████████████▍                                               | 2267/3996 [2:49:50<2:01:54,  4.23s/it]
 57%|███████████████████████████████���██████████████████████████████▍                                               | 2268/3996 [2:49:54<2:01:33,  4.22s/it]
 57%|██████████████████████████████████████████████████████████████▍                                               | 2269/3996 [2:49:59<2:05:54,  4.37s/it]
 57%|██████████████████████████████████████████████████████████████▍                                               | 2270/3996 [2:50:03<2:04:20,  4.32s/it]
 57%|██████████████████████████████████████████████████████████████▌                                               | 2271/3996 [2:50:07<2:03:06,  4.28s/it]
 57%|██████████████████████████████████████████████████████████████▌                                               | 2272/3996 [2:50:11<2:02:15,  4.26s/it]
 57%|██████████████████████████████████████████████████████████████▌                                               | 2273/3996 [2:50:15<2:01:45,  4.24s/it]
 57%|██████████████████████████████████████████████████████████████▌                                               | 2274/3996 [2:50:20<2:01:23,  4.23s/it]
 57%|██████████████████████████████████████████████████████████████▋                                               | 2275/3996 [2:50:24<2:01:00,  4.22s/it]
                                                                                                                                                           
{'loss': 0.4925, 'grad_norm': 0.2056591659784317, 'learning_rate': 8.187688599713333e-05, 'ppl': 1.6364, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4256.41, 'total_tokens': 44393451, 'epoch': 1.71}
+
 57%|██████████████████████████████████████████████████████████████▋                                               | 2275/3996 [2:50:24<2:01:00,  4.22s/it]
 57%|██████████████████████████████████████████████████████████████▋                                               | 2276/3996 [2:50:29<2:05:28,  4.38s/it]
 57%|██████████████████████████████████████████████████████████████▋                                               | 2277/3996 [2:50:33<2:03:58,  4.33s/it]
 57%|██████████████████████████████████████████████████████████████▋                                               | 2278/3996 [2:50:37<2:02:41,  4.29s/it]
 57%|██████████████████████████████████████████████████████████████▋                                               | 2279/3996 [2:50:41<2:01:49,  4.26s/it]
 57%|██████████████████████████████████████████████████████████████▊                                               | 2280/3996 [2:50:45<2:01:16,  4.24s/it]
 57%|██████████████████████████████████████████████████████████████▊                                               | 2281/3996 [2:50:50<2:00:45,  4.23s/it]
 57%|██████████████████████████████████████████████████████████████▊                                               | 2282/3996 [2:50:54<2:00:30,  4.22s/it]
 57%|██████████████████████████████████████████████████████████████▊                                               | 2283/3996 [2:50:59<2:04:57,  4.38s/it]
 57%|██████████████████████████████████████████████████████████████▊                                               | 2284/3996 [2:51:03<2:03:29,  4.33s/it]
 57%|██████████████████████████████████████████████████████████████▉                                               | 2285/3996 [2:51:07<2:02:44,  4.30s/it]
 57%|██████████████████████████████████████████████████████████████▉                                               | 2286/3996 [2:51:11<2:01:45,  4.27s/it]
 57%|██████████████████████████████████████████████████████████████▉                                               | 2287/3996 [2:51:15<2:01:07,  4.25s/it]
 57%|██████████████████████████████████████████████████████████████▉                                               | 2288/3996 [2:51:20<2:07:25,  4.48s/it]
 57%|███████████████████████████████████████████████████████████████                                               | 2289/3996 [2:51:25<2:05:00,  4.39s/it]
 57%|███████████████████████████████████████████████████████████████                                               | 2290/3996 [2:51:29<2:07:52,  4.50s/it]
 57%|███████████████████████████████████████████████████████████████                                               | 2291/3996 [2:51:34<2:05:15,  4.41s/it]
 57%|███████████████████████████████████████████████████████████████                                               | 2292/3996 [2:51:38<2:03:18,  4.34s/it]
 57%|███████████████████████████████████████████████████████████████                                               | 2293/3996 [2:51:42<2:01:51,  4.29s/it]
 57%|███████████████████████████████████████████████████████████████▏                                              | 2294/3996 [2:51:46<2:00:54,  4.26s/it]
 57%|███████████████████████████████████████████████████████████████▏                                              | 2295/3996 [2:51:50<2:00:11,  4.24s/it]
 57%|███████████████████████████████████████████████████████████████▏                                              | 2296/3996 [2:51:54<1:59:44,  4.23s/it]
 57%|███████████████████████████████████████████████████████████████▏                                              | 2297/3996 [2:51:59<2:03:55,  4.38s/it]
 58%|███████████████████████████████████████████████████████████████▎                                              | 2298/3996 [2:52:03<2:02:16,  4.32s/it]
 58%|███████████████████████████████████████████████████████████████▎                                              | 2299/3996 [2:52:08<2:01:00,  4.28s/it]
 58%|███████████████████████████████████████████████████████████████▎                                              | 2300/3996 [2:52:12<2:00:14,  4.25s/it]
                                                                                                                                                           
{'loss': 0.4952, 'grad_norm': 0.19774597883224487, 'learning_rate': 7.989817600952376e-05, 'ppl': 1.6408, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4224.5, 'total_tokens': 44836590, 'epoch': 1.73}
+
 58%|███████████████████████████████████████████████████████████████▎                                              | 2300/3996 [2:52:12<2:00:14,  4.25s/it]
 58%|███████████████████████████████████████████████████████████████▎                                              | 2301/3996 [2:52:16<1:59:48,  4.24s/it]
 58%|███████████████████████████████████████████████████████████████▎                                              | 2302/3996 [2:52:20<1:59:11,  4.22s/it]
 58%|███████████████████████████████████████████████████████████████▍                                              | 2303/3996 [2:52:24<1:59:01,  4.22s/it]
 58%|███████████████████████████████████████████████████████████████▍                                              | 2304/3996 [2:52:29<2:03:15,  4.37s/it]
 58%|███████████████████████████████████████████████████████████████▍                                              | 2305/3996 [2:52:33<2:01:51,  4.32s/it]
 58%|███████████████████████████████████████████████████████████████▍                                              | 2306/3996 [2:52:38<2:02:30,  4.35s/it]
 58%|███████████████████████████████████████████████████████████████▌                                              | 2307/3996 [2:52:42<2:01:00,  4.30s/it]
 58%|███████████████████████████████████████████████████████████████▌                                              | 2308/3996 [2:52:46<2:00:10,  4.27s/it]
 58%|███████████████████████████████████████████████████████████████▌                                              | 2309/3996 [2:52:50<1:59:30,  4.25s/it]
 58%|███████████████████████████████████████████████████████████████▌                                              | 2310/3996 [2:52:55<1:59:02,  4.24s/it]
 58%|███████████████████████████████████████████████████████████████▌                                              | 2311/3996 [2:52:59<2:03:09,  4.39s/it]
 58%|███████████████████████████████████████████████████████████████▋                                              | 2312/3996 [2:53:03<2:01:26,  4.33s/it]
 58%|███████████████████████████████████████████████████████████████▋                                              | 2313/3996 [2:53:08<2:00:06,  4.28s/it]
 58%|███████████████████████████████████████████████████████████████▋                                              | 2314/3996 [2:53:12<1:59:14,  4.25s/it]
 58%|███████████████████████████████████████████████████████████████▋                                              | 2315/3996 [2:53:16<1:58:44,  4.24s/it]
 58%|███████████████████████████████████████████████████████████████▊                                              | 2316/3996 [2:53:20<1:58:10,  4.22s/it]
 58%|███████████████████████████████████████���███████████████████████▊                                              | 2317/3996 [2:53:24<1:57:55,  4.21s/it]
 58%|███████████████████████████████████████████████████████████████▊                                              | 2318/3996 [2:53:29<2:02:11,  4.37s/it]
 58%|███████████████████████████████████████████████████████████████▊                                              | 2319/3996 [2:53:33<2:00:37,  4.32s/it]
 58%|███████████████████████████████████████████████████████████████▊                                              | 2320/3996 [2:53:37<1:59:22,  4.27s/it]
 58%|███████████████████████████████████████████████████████████████▉                                              | 2321/3996 [2:53:42<1:58:39,  4.25s/it]
 58%|███████████████████████████████████████████████████████████████▉                                              | 2322/3996 [2:53:46<1:58:06,  4.23s/it]
 58%|███████████████████████████████████████████████████████████████▉                                              | 2323/3996 [2:53:50<1:57:39,  4.22s/it]
 58%|███████████████████████████████████████████████████████████████▉                                              | 2324/3996 [2:53:54<1:57:25,  4.21s/it]
 58%|████████████████████████████████████████████████████████████████                                              | 2325/3996 [2:53:59<2:01:44,  4.37s/it]
                                                                                                                                                           
{'loss': 0.4977, 'grad_norm': 0.19662383198738098, 'learning_rate': 7.792763490394984e-05, 'ppl': 1.6449, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3741.52, 'total_tokens': 45279799, 'epoch': 1.74}
+
 58%|████████████████████████████████████████████████████████████████                                              | 2325/3996 [2:53:59<2:01:44,  4.37s/it]
 58%|████████████████████████████████████████████████████████████████                                              | 2326/3996 [2:54:03<2:00:11,  4.32s/it]
 58%|████████████████████████████████████████████████████████████████                                              | 2327/3996 [2:54:07<1:59:00,  4.28s/it]
 58%|████████████████████████████████████████████████████████████████                                              | 2328/3996 [2:54:12<1:58:07,  4.25s/it]
 58%|████████████████████████████████████████████████████████████████                                              | 2329/3996 [2:54:16<1:57:43,  4.24s/it]
 58%|████████████████████████████████████████████████████████████████▏                                             | 2330/3996 [2:54:20<1:57:18,  4.22s/it]
 58%|████████████████████████████████████████████████████████████████▏                                             | 2331/3996 [2:54:24<1:57:07,  4.22s/it]
 58%|████████████████████████████████████████████████████████████████▏                                             | 2332/3996 [2:54:29<2:01:18,  4.37s/it]
 58%|████████████████████████████████████████████████████████████████▏                                             | 2333/3996 [2:54:33<1:59:43,  4.32s/it]
 58%|████████████████████████████████████████████████████████████████▏                                             | 2334/3996 [2:54:37<1:58:33,  4.28s/it]
 58%|████████████████████████████████████████████████████████████████▎                                             | 2335/3996 [2:54:41<1:57:45,  4.25s/it]
 58%|████████████████████████████████████████████████████████████████▎                                             | 2336/3996 [2:54:46<1:57:12,  4.24s/it]
 58%|████████████████████████████████████████████████████████████████▎                                             | 2337/3996 [2:54:50<1:56:44,  4.22s/it]
 59%|████████████████████████████████████████████████████████████████▎                                             | 2338/3996 [2:54:54<1:56:34,  4.22s/it]
 59%|████████████████████████████████████████████████████████████████▍                                             | 2339/3996 [2:54:59<2:00:46,  4.37s/it]
 59%|████████████████████████████████████████████████████████████████▍                                             | 2340/3996 [2:55:03<1:59:10,  4.32s/it]
 59%|████████████████████████████████████████████████████████████████▍                                             | 2341/3996 [2:55:07<1:57:52,  4.27s/it]
 59%|████████████████████████████████████████████████████████████████▍                                             | 2342/3996 [2:55:12<1:59:26,  4.33s/it]
 59%|████████████████████████████████████████████████████████████████▍                                             | 2343/3996 [2:55:16<1:58:19,  4.29s/it]
 59%|████████████████████████████████████████████████████████████████▌                                             | 2344/3996 [2:55:20<1:57:29,  4.27s/it]
 59%|████████████████████████████████████████████████████████████████▌                                             | 2345/3996 [2:55:24<1:56:49,  4.25s/it]
 59%|████████████████████████████████████████████████████████████████▌                                             | 2346/3996 [2:55:29<2:00:52,  4.40s/it]
 59%|████████████████████████████████████████████████████████████████▌                                             | 2347/3996 [2:55:33<1:59:06,  4.33s/it]
 59%|████████████████████████████████████████████████████████████████▋                                             | 2348/3996 [2:55:37<1:57:51,  4.29s/it]
 59%|████████████████████████████████████████████████████████████████▋                                             | 2349/3996 [2:55:42<1:56:53,  4.26s/it]
 59%|████████████████████████████████████████████████████████████████▋                                             | 2350/3996 [2:55:46<1:56:19,  4.24s/it]
                                                                                                                                                           
{'loss': 0.4965, 'grad_norm': 0.19400179386138916, 'learning_rate': 7.596606345937812e-05, 'ppl': 1.643, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4248.51, 'total_tokens': 45725602, 'epoch': 1.76}
+
 59%|████████████████████████████████████████████████████████████████▋                                             | 2350/3996 [2:55:46<1:56:19,  4.24s/it]
 59%|████████████████████████████████████████████████████████████████▋                                             | 2351/3996 [2:55:50<1:55:42,  4.22s/it]
 59%|████████████████████████████████████████████████████████████████▋                                             | 2352/3996 [2:55:54<1:55:30,  4.22s/it]
 59%|████████████████████████████████████████████████████████████████▊                                             | 2353/3996 [2:55:59<1:59:45,  4.37s/it]
 59%|████████████████████████████████████████████████████████████████▊                                             | 2354/3996 [2:56:03<1:58:09,  4.32s/it]
 59%|████████████████████████████████████████████████████████████████▊                                             | 2355/3996 [2:56:07<1:57:02,  4.28s/it]
 59%|████████████████████████████████████████████████████████████████▊                                             | 2356/3996 [2:56:11<1:56:06,  4.25s/it]
 59%|████████████████████████████████████████████████████████████████▉                                             | 2357/3996 [2:56:16<1:55:36,  4.23s/it]
 59%|████████████████████████████████████████████████████████████████▉                                             | 2358/3996 [2:56:20<1:55:07,  4.22s/it]
 59%|████████████████████████████████████████████████████████████████▉                                             | 2359/3996 [2:56:24<1:54:58,  4.21s/it]
 59%|████████████████████████████████████████████████████████████████▉                                             | 2360/3996 [2:56:29<1:59:08,  4.37s/it]
 59%|████████████████████████████████████████████████████████████████▉                                             | 2361/3996 [2:56:33<1:57:36,  4.32s/it]
 59%|█████████████████████████████████████████████████████████████████                                             | 2362/3996 [2:56:37<1:56:25,  4.28s/it]
 59%|█████████████████████████████████████████████████████████████████                                             | 2363/3996 [2:56:41<1:55:30,  4.24s/it]
 59%|█████████████████████████████████████████████████████████████████                                             | 2364/3996 [2:56:45<1:55:12,  4.24s/it]
 59%|█████████████████████████████████████████████████████████████████                                             | 2365/3996 [2:56:50<1:54:39,  4.22s/it]
 59%|████���████████████████████████████████████████████████████████████▏                                            | 2366/3996 [2:56:54<1:54:31,  4.22s/it]
 59%|█████████████████████████████████████████████████████████████████▏                                            | 2367/3996 [2:56:59<1:58:44,  4.37s/it]
 59%|█████████████████████████████████████████████████████████████████▏                                            | 2368/3996 [2:57:03<1:57:18,  4.32s/it]
 59%|█████████████████████████████████████████████████████████████████▏                                            | 2369/3996 [2:57:07<1:55:59,  4.28s/it]
 59%|█████████████████████████████████████████████████████████████████▏                                            | 2370/3996 [2:57:11<1:55:10,  4.25s/it]
 59%|█████████████████████████████████████████████████████████████████▎                                            | 2371/3996 [2:57:15<1:54:44,  4.24s/it]
 59%|█████████████████████████████████████████████████████████████████▎                                            | 2372/3996 [2:57:20<1:54:13,  4.22s/it]
 59%|█████████████████████████████████████████████████████████████████▎                                            | 2373/3996 [2:57:24<1:54:01,  4.22s/it]
 59%|█████████████████████████████████████████████████████████████████▎                                            | 2374/3996 [2:57:29<1:58:07,  4.37s/it]
 59%|█████████████████████████████████████████████████████████████████▍                                            | 2375/3996 [2:57:33<1:56:38,  4.32s/it]
                                                                                                                                                           
{'loss': 0.5014, 'grad_norm': 0.20261766016483307, 'learning_rate': 7.401425880972742e-05, 'ppl': 1.651, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4216.2, 'total_tokens': 46167730, 'epoch': 1.78}
+
 59%|█████████████████████████████████████████████████████████████████▍                                            | 2375/3996 [2:57:33<1:56:38,  4.32s/it]
 59%|█████████████████████████████████████████████████████████████████▍                                            | 2376/3996 [2:57:37<1:55:32,  4.28s/it]
 59%|█████████████████████████████████████████████████████████████████▍                                            | 2377/3996 [2:57:41<1:54:33,  4.25s/it]
 60%|█████████████████████████████████████████████████████████████████▍                                            | 2378/3996 [2:57:45<1:54:10,  4.23s/it]
 60%|█████████████████████████████████████████████████████████████████▍                                            | 2379/3996 [2:57:49<1:53:45,  4.22s/it]
 60%|█████████████████████████████████████████████████████████████████▌                                            | 2380/3996 [2:57:54<1:53:21,  4.21s/it]
 60%|████████████████████████���████████████████████████████████████████▌                                            | 2381/3996 [2:57:58<1:57:33,  4.37s/it]
 60%|█████████████████████████████████████████████████████████████████▌                                            | 2382/3996 [2:58:03<1:56:02,  4.31s/it]
 60%|█████████████████████████████████████████████████████████████████▌                                            | 2383/3996 [2:58:07<1:55:00,  4.28s/it]
 60%|█████████████████████████████████████████████████████████████████▋                                            | 2384/3996 [2:58:11<1:54:04,  4.25s/it]
 60%|█████████████████████████████████████████████████████████████████▋                                            | 2385/3996 [2:58:15<1:53:40,  4.23s/it]
 60%|█████████████████████████████████████████████████████████████████▋                                            | 2386/3996 [2:58:19<1:53:13,  4.22s/it]
 60%|█████████████████████████████████████████████████████████████████▋                                            | 2387/3996 [2:58:24<1:52:59,  4.21s/it]
 60%|█████████████████████████████████████████████████████████████████▋                                            | 2388/3996 [2:58:28<1:57:07,  4.37s/it]
 60%|█████████████████████████████████████████████████████████████████▊                                            | 2389/3996 [2:58:32<1:55:41,  4.32s/it]
 60%|█████████████████████████████████████████████████████████████████▊                                            | 2390/3996 [2:58:37<1:54:30,  4.28s/it]
 60%|█████████████████████████████████████████████████████████████████▊                                            | 2391/3996 [2:58:41<1:53:38,  4.25s/it]
 60%|█████████████████████████████████████████████████████████████████▊                                            | 2392/3996 [2:58:45<1:53:10,  4.23s/it]
 60%|█████████████████████████████████████████████████████████████████▊                                            | 2393/3996 [2:58:50<1:55:34,  4.33s/it]
 60%|█████████████████████████████████████████████████████████████████▉                                            | 2394/3996 [2:58:54<1:54:34,  4.29s/it]
 60%|█████████████████████████████████████████████████████████████████▉                                            | 2395/3996 [2:58:59<1:58:10,  4.43s/it]
 60%|█████████████████████████████████████████████████████████████████▉                                            | 2396/3996 [2:59:03<1:58:07,  4.43s/it]
 60%|█████████████████████████████████████████████████████████████████▉                                            | 2397/3996 [2:59:07<1:55:57,  4.35s/it]
 60%|██████████████████████████████████████████████████████████████████                                            | 2398/3996 [2:59:11<1:54:36,  4.30s/it]
 60%|██████████████████████████████████████████████████████████████████                                            | 2399/3996 [2:59:16<1:53:50,  4.28s/it]
 60%|██████████████████████████████████████████████████████████████████                                            | 2400/3996 [2:59:20<1:57:36,  4.42s/it]
                                                                                                                                                           
{'loss': 0.4901, 'grad_norm': 0.20447255671024323, 'learning_rate': 7.207301411993387e-05, 'ppl': 1.6325, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3727.37, 'total_tokens': 46611126, 'epoch': 1.8}
+
 60%|██████████████████████████████████████████████████████████████████                                            | 2400/3996 [2:59:20<1:57:36,  4.42s/it]
 60%|██████████████████████████████████████████████████████████████████                                            | 2401/3996 [2:59:24<1:55:41,  4.35s/it]
 60%|██████████████████████████████████████████████████████████████████                                            | 2402/3996 [2:59:29<1:58:32,  4.46s/it]
 60%|██████████████████████████████████████████████████████████████████▏                                           | 2403/3996 [2:59:33<1:56:15,  4.38s/it]
 60%|██████████████████████████████████████████████████████████████████▏                                           | 2404/3996 [2:59:38<1:54:44,  4.32s/it]
 60%|██████████████████████████████████████████████████████████████████▏                                           | 2405/3996 [2:59:42<1:53:31,  4.28s/it]
 60%|██████████████████████████████████████████████████████████████████▏                                           | 2406/3996 [2:59:46<1:52:48,  4.26s/it]
 60%|██████████████████████████████████████████████████████████████████▎                                           | 2407/3996 [2:59:50<1:52:08,  4.23s/it]
 60%|██████████████████████████████████████████████████████████████████▎                                           | 2408/3996 [2:59:54<1:51:44,  4.22s/it]
 60%|██████████████████████████████████████████████████████████████████▎                                           | 2409/3996 [2:59:59<1:55:42,  4.37s/it]
 60%|██████████████████████████████████████████████████████████████████▎                                           | 2410/3996 [3:00:03<1:54:17,  4.32s/it]
 60%|██████████████████████████████████████████████████████████████████▎                                           | 2411/3996 [3:00:07<1:53:01,  4.28s/it]
 60%|██████████████████████████████████████████████████████████████████▍                                           | 2412/3996 [3:00:12<1:52:17,  4.25s/it]
 60%|██████████████████████████████████████████████████████████████████▍                                           | 2413/3996 [3:00:16<1:51:49,  4.24s/it]
 60%|███████████████���██████████████████████████████████████████████████▍                                           | 2414/3996 [3:00:20<1:51:19,  4.22s/it]
 60%|██████████████████████████████████████████████████████████████████▍                                           | 2415/3996 [3:00:24<1:51:03,  4.21s/it]
 60%|██████████████████████████████████████████████████████████████████▌                                           | 2416/3996 [3:00:29<1:55:02,  4.37s/it]
 60%|██████████████████████████████████████████████████████████████████▌                                           | 2417/3996 [3:00:33<1:53:37,  4.32s/it]
 61%|██████████████████████████████████████████████████████████████████▌                                           | 2418/3996 [3:00:37<1:52:37,  4.28s/it]
 61%|██████████████████████████████████████████████████████████████████▌                                           | 2419/3996 [3:00:42<1:51:53,  4.26s/it]
 61%|██████████████████████████████████████████████████████████████████▌                                           | 2420/3996 [3:00:46<1:51:23,  4.24s/it]
 61%|██████████████████████████████████████████████████████████████████▋                                           | 2421/3996 [3:00:50<1:50:50,  4.22s/it]
 61%|██████████████████████████████████████████████████████████████████▋                                           | 2422/3996 [3:00:54<1:50:32,  4.21s/it]
 61%|██████████████████████████████████████████████████████████████████▋                                           | 2423/3996 [3:00:59<1:54:36,  4.37s/it]
 61%|██████████████████████████████████████████████████████████████████▋                                           | 2424/3996 [3:01:03<1:53:08,  4.32s/it]
 61%|██████████████████████████████████████████████████████████████████▊                                           | 2425/3996 [3:01:07<1:52:01,  4.28s/it]
                                                                                                                                                           
{'loss': 0.4925, 'grad_norm': 0.19921696186065674, 'learning_rate': 7.014311826362804e-05, 'ppl': 1.6364, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4202.19, 'total_tokens': 47050763, 'epoch': 1.82}
+
 61%|██████████████████████████████████████████████████████████████████▊                                           | 2425/3996 [3:01:07<1:52:01,  4.28s/it]
 61%|██████████████████████████████████████████████████████████████████▊                                           | 2426/3996 [3:01:11<1:51:10,  4.25s/it]
 61%|██████████████████████████████████████████████████████████████████▊                                           | 2427/3996 [3:01:16<1:50:37,  4.23s/it]
 61%|██████████████████████████████████████████████████████████████████▊                                           | 2428/3996 [3:01:20<1:50:16,  4.22s/it]
 61%|██████████████████████████████████████████████████████████████████▊                                           | 2429/3996 [3:01:24<1:50:01,  4.21s/it]
 61%|██████████████████████████████████████████████████████████████████▉                                           | 2430/3996 [3:01:29<1:53:56,  4.37s/it]
 61%|██████████████████████████████████████████████████████████████████▉                                           | 2431/3996 [3:01:33<1:52:35,  4.32s/it]
 61%|██████████████████████████████████████████████████████████████████▉                                           | 2432/3996 [3:01:37<1:51:30,  4.28s/it]
 61%|██████████████████████████████████████████████████████████████████▉                                           | 2433/3996 [3:01:41<1:50:44,  4.25s/it]
 61%|███████████████████████████████████████████████████████████████████                                           | 2434/3996 [3:01:46<1:50:20,  4.24s/it]
 61%|███████████████████████████████████████████████████████████████████                                           | 2435/3996 [3:01:50<1:49:51,  4.22s/it]
 61%|███████████████████████████████████████████████████████████████████                                           | 2436/3996 [3:01:54<1:49:39,  4.22s/it]
 61%|███████████████████████████████████████████████████████████████████                                           | 2437/3996 [3:01:59<1:53:40,  4.37s/it]
 61%|███████████████████████████████████████████████████████████████████                                           | 2438/3996 [3:02:03<1:52:16,  4.32s/it]
 61%|███████████████████████████████████████████████████████████████████▏                                          | 2439/3996 [3:02:07<1:51:02,  4.28s/it]
 61%|███████████████████████████████████████████████████████████████████▏                                          | 2440/3996 [3:02:11<1:50:14,  4.25s/it]
 61%|███████████████████████████████████████████████████████████████████▏                                          | 2441/3996 [3:02:15<1:49:46,  4.24s/it]
 61%|███████████████████████████████████████████████████████████████████▏                                          | 2442/3996 [3:02:20<1:49:22,  4.22s/it]
 61%|███████████████████████████████████████████████████████████████████▏                                          | 2443/3996 [3:02:24<1:49:08,  4.22s/it]
 61%|███████████████████████████████████████████████████████████████████▎                                          | 2444/3996 [3:02:29<1:53:10,  4.38s/it]
 61%|███████████████████████████████████████████████████████████████████▎                                          | 2445/3996 [3:02:33<1:51:50,  4.33s/it]
 61%|███████████████████████████████████████████████████████████████████▎                                          | 2446/3996 [3:02:37<1:50:48,  4.29s/it]
 61%|███████████████████████████████████████████████████████████████████▎                                          | 2447/3996 [3:02:41<1:49:53,  4.26s/it]
 61%|███████████████████████████████████████████████████████████████████▍                                          | 2448/3996 [3:02:45<1:49:22,  4.24s/it]
 61%|███████████████████████████████████████████████████████████████████▍                                          | 2449/3996 [3:02:50<1:48:50,  4.22s/it]
 61%|███████████████████████████████████████████████████████████████████▍                                          | 2450/3996 [3:02:54<1:48:45,  4.22s/it]
                                                                                                                                                           
{'loss': 0.494, 'grad_norm': 0.20095540583133698, 'learning_rate': 6.822535550255652e-05, 'ppl': 1.6389, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4230.16, 'total_tokens': 47496926, 'epoch': 1.84}
+
 61%|███████████████████████████████████████████████████████████████████▍                                          | 2450/3996 [3:02:54<1:48:45,  4.22s/it]
 61%|███████████████████████████████████████████████████████████████████▍                                          | 2451/3996 [3:02:59<1:53:02,  4.39s/it]
 61%|███████████████████████████████████████████████████████████████████▍                                          | 2452/3996 [3:03:03<1:51:29,  4.33s/it]
 61%|███████████████████████████████████████████████████████████████████▌                                          | 2453/3996 [3:03:07<1:50:13,  4.29s/it]
 61%|███████████████████████████████████████████████████████████████████▌                                          | 2454/3996 [3:03:11<1:49:22,  4.26s/it]
 61%|███████████████████████████████████████████████████████████████████▌                                          | 2455/3996 [3:03:15<1:48:52,  4.24s/it]
 61%|███████████████████████████████████████████████████████████████████▌                                          | 2456/3996 [3:03:19<1:48:28,  4.23s/it]
 61%|███████████████████████████████████████████████████████████████████▋                                          | 2457/3996 [3:03:24<1:48:10,  4.22s/it]
 62%|███████████████████████████████████████████████████████████████████▋                                          | 2458/3996 [3:03:28<1:52:05,  4.37s/it]
 62%|███████████████████████████████████████████████████████████████████▋                                          | 2459/3996 [3:03:33<1:50:42,  4.32s/it]
 62%|███████████████████████████████████████████████████████████████████▋                                          | 2460/3996 [3:03:37<1:49:40,  4.28s/it]
 62%|███████████████████████████████████████████████████████████████████▋                                          | 2461/3996 [3:03:41<1:48:47,  4.25s/it]
 62%|███████████████████████████████████████████████████████████████████▊                                          | 2462/3996 [3:03:45<1:48:22,  4.24s/it]
 62%|███████████████████████████████████████████████████████████████████▊                                          | 2463/3996 [3:03:49<1:47:50,  4.22s/it]
 62%|███████████████████████████████████████████████████████████████████▊                                          | 2464/3996 [3:03:54<1:47:36,  4.21s/it]
 62%|███████████████████████████████████████████████████████████████████▊                                          | 2465/3996 [3:03:58<1:51:29,  4.37s/it]
 62%|███████████████████████████████████████████████████████████████████▉                                          | 2466/3996 [3:04:03<1:50:10,  4.32s/it]
 62%|███████████████████████████████████████████████████████████████████▉                                          | 2467/3996 [3:04:07<1:49:02,  4.28s/it]
 62%|███████████████████████████████████████████████████████████████████▉                                          | 2468/3996 [3:04:11<1:48:23,  4.26s/it]
 62%|███████████████████████████████████████████████████████████████████▉                                          | 2469/3996 [3:04:15<1:49:59,  4.32s/it]
 62%|███████████████████████████████████████████████████████████████████▉                                          | 2470/3996 [3:04:20<1:48:53,  4.28s/it]
 62%|████████████████████████████████████████████████████████████████████                                          | 2471/3996 [3:04:24<1:48:17,  4.26s/it]
 62%|████████████████████████████████████████████████████████████████████                                          | 2472/3996 [3:04:29<1:51:51,  4.40s/it]
 62%|████████████████████████████████████████████████████████████████████                                          | 2473/3996 [3:04:33<1:50:14,  4.34s/it]
 62%|████████████████████████████████████████████████████████████████████                                          | 2474/3996 [3:04:37<1:49:06,  4.30s/it]
 62%|████████████████████████████████████████████████████████████████████▏                                         | 2475/3996 [3:04:41<1:48:05,  4.26s/it]
                                                                                                                                                           
{'loss': 0.5036, 'grad_norm': 0.20210741460323334, 'learning_rate': 6.632050516787719e-05, 'ppl': 1.6547, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4256.1, 'total_tokens': 47941250, 'epoch': 1.86}
+
 62%|████████████████████████████████████████████████████████████████████▏                                         | 2475/3996 [3:04:41<1:48:05,  4.26s/it]
 62%|████████████████████████████████████████████████████████████████████▏                                         | 2476/3996 [3:04:45<1:47:31,  4.24s/it]
 62%|████████████████████████████████████████████████████████████████████▏                                         | 2477/3996 [3:04:49<1:47:00,  4.23s/it]
 62%|████████████████████████████████████████████████████████████████████▏                                         | 2478/3996 [3:04:54<1:46:40,  4.22s/it]
 62%|████████████████████████████████████████████████████████████████████▏                                         | 2479/3996 [3:04:58<1:50:33,  4.37s/it]
 62%|████████████████████████████████████████████████████████████████████▎                                         | 2480/3996 [3:05:03<1:49:11,  4.32s/it]
 62%|████████████████████████████████████████████████████████████████████▎                                         | 2481/3996 [3:05:07<1:48:02,  4.28s/it]
 62%|████████████████████████████████████████████████████████████████████▎                                         | 2482/3996 [3:05:11<1:47:14,  4.25s/it]
 62%|████████████████████████████████████████████████████████████████████▎                                         | 2483/3996 [3:05:15<1:46:50,  4.24s/it]
 62%|████████████████████████████████████████████████████████████████████▍                                         | 2484/3996 [3:05:19<1:46:29,  4.23s/it]
 62%|████████████████████████████████████████████████████████████████████▍                                         | 2485/3996 [3:05:24<1:46:15,  4.22s/it]
 62%|████████████████████████████████████████████████████████████████████▍                                         | 2486/3996 [3:05:28<1:51:15,  4.42s/it]
 62%|████████████████████████████████████████████████████████████████████▍                                         | 2487/3996 [3:05:33<1:49:31,  4.36s/it]
 62%|████████████████████████████████████████████████████████████████████▍                                         | 2488/3996 [3:05:37<1:48:16,  4.31s/it]
 62%|████████████████████████████████████████████████████████████████████▌                                         | 2489/3996 [3:05:41<1:47:15,  4.27s/it]
 62%|████████████████████████████████████████████████████████████████████▌                                         | 2490/3996 [3:05:45<1:46:38,  4.25s/it]
 62%|████████████████████████████████████████████████████████████████████▌                                         | 2491/3996 [3:05:49<1:46:05,  4.23s/it]
 62%|████████████████████████████████████████████████████████████████████▌                                         | 2492/3996 [3:05:54<1:45:46,  4.22s/it]
 62%|████████████████████████████████████████████████████████████████████▋                                         | 2493/3996 [3:05:58<1:49:32,  4.37s/it]
 62%|████████████████████��███████████████████████████████████████████████▋                                         | 2494/3996 [3:06:03<1:48:06,  4.32s/it]
 62%|████████████████████████████████████████████████████████████████████▋                                         | 2495/3996 [3:06:07<1:47:06,  4.28s/it]
 62%|████████████████████████████████████████████████████████████████████▋                                         | 2496/3996 [3:06:11<1:46:24,  4.26s/it]
 62%|████████████████████████████████████████████████████████████████████▋                                         | 2497/3996 [3:06:15<1:45:58,  4.24s/it]
 63%|████████████████████████████████████████████████████████████████████▊                                         | 2498/3996 [3:06:19<1:45:28,  4.22s/it]
 63%|████████████████████████████████████████████████████████████████████▊                                         | 2499/3996 [3:06:24<1:45:10,  4.22s/it]
 63%|████████████████████████████████████████████████████████████████████▊                                         | 2500/3996 [3:06:28<1:49:03,  4.37s/it]
                                                                                                                                                           
{'loss': 0.5019, 'grad_norm': 0.21025419235229492, 'learning_rate': 6.442934134345871e-05, 'ppl': 1.6519, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3728.09, 'total_tokens': 48383306, 'epoch': 1.88}
+
 63%|████████████████████████████████████████████████████████████████████▊                                         | 2500/3996 [3:06:28<1:49:03,  4.37s/it]
 63%|████████████████████████████████████████████████████████████████████▊                                         | 2501/3996 [3:06:32<1:47:40,  4.32s/it]
 63%|████████████████████████████████████████████████████████████████████▊                                         | 2502/3996 [3:06:37<1:46:36,  4.28s/it]
 63%|████████████████████████████████████████████████████████████████████▉                                         | 2503/3996 [3:06:41<1:45:43,  4.25s/it]
 63%|████████████████████████████████████████████████████████████████████▉                                         | 2504/3996 [3:06:45<1:45:20,  4.24s/it]
 63%|████████████████████████████████████████████████████████████████████▉                                         | 2505/3996 [3:06:49<1:44:56,  4.22s/it]
 63%|████████████████████████████████████████████████████████████████████▉                                         | 2506/3996 [3:06:53<1:44:43,  4.22s/it]
 63%|█████████████████████████████████████████████████████████████████████                                         | 2507/3996 [3:06:58<1:48:28,  4.37s/it]
 63%|█████████████████████████████████████████████████████████████████████                                         | 2508/3996 [3:07:02<1:47:17,  4.33s/it]
 63%|█████████████████████████████████████████████████████████████████████                                         | 2509/3996 [3:07:07<1:46:09,  4.28s/it]
 63%|█████████████████████████████████████████████████████████████████████                                         | 2510/3996 [3:07:11<1:45:20,  4.25s/it]
 63%|█████████████████████████████████████████████████████████████████████                                         | 2511/3996 [3:07:15<1:44:57,  4.24s/it]
 63%|█████████████████████████████████████████████████████████████████████▏                                        | 2512/3996 [3:07:19<1:44:39,  4.23s/it]
 63%|█████████████████████████████████████████████████████████████████████▏                                        | 2513/3996 [3:07:23<1:44:18,  4.22s/it]
 63%|█████████████████████████████████████████████████████████████████████▏                                        | 2514/3996 [3:07:28<1:48:03,  4.37s/it]
 63%|█████████████████████████████████████████████████████████████████████▏                                        | 2515/3996 [3:07:32<1:46:40,  4.32s/it]
 63%|█████████████████████████████████████████████████████████████████████▎                                        | 2516/3996 [3:07:37<1:45:38,  4.28s/it]
 63%|█████████████████████████████████████████████████████████████████████▎                                        | 2517/3996 [3:07:41<1:44:50,  4.25s/it]
 63%|█████████████████████████████████████████████████████████████████████▎                                        | 2518/3996 [3:07:45<1:44:21,  4.24s/it]
 63%|█████████████████████████████████████████████████████████████████████▎                                        | 2519/3996 [3:07:49<1:43:57,  4.22s/it]
 63%|█████████████████████████████████████████████████████████████████████▎                                        | 2520/3996 [3:07:53<1:43:41,  4.22s/it]
 63%|█████████████████████████████████████████████████████████████████████▍                                        | 2521/3996 [3:07:58<1:47:31,  4.37s/it]
 63%|█████████████████████████████████████████████████████████████████████▍                                        | 2522/3996 [3:08:02<1:46:04,  4.32s/it]
 63%|█████████████████████████████████████████████████████████████████████▍                                        | 2523/3996 [3:08:06<1:45:10,  4.28s/it]
 63%|█████████████████████████████████████████████████████████████████████▍                                        | 2524/3996 [3:08:11<1:44:16,  4.25s/it]
 63%|█████████████████████████████████████████████████████████████████████▌                                        | 2525/3996 [3:08:15<1:43:49,  4.23s/it]
                                                                                                                                                           
{'loss': 0.5022, 'grad_norm': 0.20130059123039246, 'learning_rate': 6.255263255131172e-05, 'ppl': 1.6524, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4178.95, 'total_tokens': 48821862, 'epoch': 1.89}
+
 63%|█████████████████████████████████████████████████████████████████████▌                                        | 2525/3996 [3:08:15<1:43:49,  4.23s/it]
 63%|█████████████████████████████████████████████████████████████████████▌                                        | 2526/3996 [3:08:19<1:43:27,  4.22s/it]
 63%|█████████████████████████████████████████████████████████████████████▌                                        | 2527/3996 [3:08:23<1:43:08,  4.21s/it]
 63%|█████████████████████████████████████████████████████████████████████▌                                        | 2528/3996 [3:08:28<1:46:48,  4.37s/it]
 63%|█████████████████████████████████████████████████████████████████████▌                                        | 2529/3996 [3:08:32<1:45:29,  4.31s/it]
 63%|█████████████████████████████████████████████████████████████████████▋                                        | 2530/3996 [3:08:36<1:44:26,  4.27s/it]
 63%|█████████████████████████████████████████████████████████████████████▋                                        | 2531/3996 [3:08:40<1:43:39,  4.25s/it]
 63%|█████████████████████████████████████████████████████████████████████▋                                        | 2532/3996 [3:08:45<1:43:15,  4.23s/it]
 63%|█████████████████████████████████████████████████████████████████████▋                                        | 2533/3996 [3:08:49<1:43:41,  4.25s/it]
 63%|█████████████████████████████████████████████████████████████████████▊                                        | 2534/3996 [3:08:53<1:43:21,  4.24s/it]
 63%|█████████████████████████████████████████████████████████████████████▊                                        | 2535/3996 [3:08:58<1:46:58,  4.39s/it]
 63%|█████████████████████████████████████████████████████████████████████▊                                        | 2536/3996 [3:09:02<1:45:27,  4.33s/it]
 63%|█████████████████████████████████████████████████████████████████████▊                                        | 2537/3996 [3:09:06<1:44:20,  4.29s/it]
 64%|█████████████████████████████████████████████████████████████████████▊                                        | 2538/3996 [3:09:10<1:43:24,  4.26s/it]
 64%|█████████████████████████████████████████████████████████████████████▉                                        | 2539/3996 [3:09:15<1:43:01,  4.24s/it]
 64%|█████████████████████████████████████████████████████████████████████▉                                        | 2540/3996 [3:09:19<1:42:37,  4.23s/it]
 64%|█████████████████████████████████████████████████████████████████████▉                                        | 2541/3996 [3:09:23<1:42:18,  4.22s/it]
 64%|█████████████████████████████████████████████████████████████████████▉                                        | 2542/3996 [3:09:28<1:46:03,  4.38s/it]
 64%|██████████████████████████████████████████████████████████████████████                                        | 2543/3996 [3:09:32<1:45:03,  4.34s/it]
 64%|██████████████████████████████████████████████████████████████████████                                        | 2544/3996 [3:09:36<1:43:53,  4.29s/it]
 64%|██████████████████████████████████████████████████████████████████████                                        | 2545/3996 [3:09:40<1:43:06,  4.26s/it]
 64%|██████████████████████████████████████████████████████████████████████                                        | 2546/3996 [3:09:45<1:42:36,  4.25s/it]
 64%|██████████████████████████████████████████████████████████████████████                                        | 2547/3996 [3:09:49<1:42:25,  4.24s/it]
 64%|██████████████████████████████████████████████████████████████████████▏                                       | 2548/3996 [3:09:53<1:42:01,  4.23s/it]
 64%|██████████████████████████████████████████████████████████████████████▏                                       | 2549/3996 [3:09:58<1:45:39,  4.38s/it]
 64%|██████████████████████████████████████████████████████████████████████▏                                       | 2550/3996 [3:10:02<1:45:46,  4.39s/it]
                                                                                                                                                           
{'loss': 0.4876, 'grad_norm': 0.19601669907569885, 'learning_rate': 6.0691141439280785e-05, 'ppl': 1.6284, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3998.52, 'total_tokens': 49262344, 'epoch': 1.91}
+
 64%|██████████████████████████████████████████████████████████████████████▏                                       | 2550/3996 [3:10:02<1:45:46,  4.39s/it]
 64%|██████████████████████████████████████████████████████████████████████▏                                       | 2551/3996 [3:10:06<1:44:12,  4.33s/it]
 64%|██████████████████████████████████████████████████████████████████████▎                                       | 2552/3996 [3:10:11<1:43:05,  4.28s/it]
 64%|██████████████████████████████████████████████████████████████████████▎                                       | 2553/3996 [3:10:15<1:42:57,  4.28s/it]
 64%|██████████████████████████████████████████████████████████████████████▎                                       | 2554/3996 [3:10:19<1:42:19,  4.26s/it]
 64%|██████████████████████████████████████████████████████████████████████▎                                       | 2555/3996 [3:10:23<1:41:54,  4.24s/it]
 64%|██████████████████████████████████████████████████████████████████████▎                                       | 2556/3996 [3:10:28<1:45:21,  4.39s/it]
 64%|██████████████████████████████████████████████████████████████████████▍                                       | 2557/3996 [3:10:32<1:43:56,  4.33s/it]
 64%|██████████████████████████████████████████████████████████████████████▍                                       | 2558/3996 [3:10:36<1:42:55,  4.29s/it]
 64%|██████████████████████████████████████████████████████████████████████▍                                       | 2559/3996 [3:10:41<1:44:23,  4.36s/it]
 64%|██████████████████████████████████████████████████████████████████████▍                                       | 2560/3996 [3:10:45<1:43:19,  4.32s/it]
 64%|██████████████████████████████████████████████████████████████████████▍                                       | 2561/3996 [3:10:49<1:42:16,  4.28s/it]
 64%|██████████████████████████████████████████████████████████████████████▌                                       | 2562/3996 [3:10:54<1:41:37,  4.25s/it]
 64%|██████████████████████████████████████████████████████████████████████▌                                       | 2563/3996 [3:10:58<1:45:00,  4.40s/it]
 64%|██████████████████████████████████████████████████████████████████████▌                                       | 2564/3996 [3:11:02<1:43:36,  4.34s/it]
 64%|██████████████████████████████████████████████████████████████████████▌                                       | 2565/3996 [3:11:07<1:42:29,  4.30s/it]
 64%|██████████████████████████████████████████████████████████████████████▋                                       | 2566/3996 [3:11:11<1:41:38,  4.26s/it]
 64%|██████████████████████████████████████████████████████████████████████▋                                       | 2567/3996 [3:11:15<1:41:11,  4.25s/it]
 64%|██████████████████████████████████████████████████████████████████████▋                                       | 2568/3996 [3:11:19<1:40:39,  4.23s/it]
 64%|██████████████████████████████████████████████████████████████████████▋                                       | 2569/3996 [3:11:23<1:40:23,  4.22s/it]
 64%|██████████████████████████████████████████████████████████████████████▋                                       | 2570/3996 [3:11:28<1:43:55,  4.37s/it]
 64%|██████████████████████████████████████████████████████████████████████▊                                       | 2571/3996 [3:11:32<1:42:41,  4.32s/it]
 64%|██████████████████████████████████████████████████████████████████████▊                                       | 2572/3996 [3:11:37<1:41:42,  4.29s/it]
 64%|████���█████████████████████████████████████████████████████████████████▊                                       | 2573/3996 [3:11:41<1:40:55,  4.26s/it]
 64%|██████████████████████████████████████████████████████████████████████▊                                       | 2574/3996 [3:11:45<1:40:33,  4.24s/it]
 64%|██████████████████████████████████████████████████████████████████████▉                                       | 2575/3996 [3:11:49<1:40:10,  4.23s/it]
                                                                                                                                                           
{'loss': 0.4796, 'grad_norm': 0.20538586378097534, 'learning_rate': 5.884562447112331e-05, 'ppl': 1.6154, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4192.8, 'total_tokens': 49702209, 'epoch': 1.93}
+
 64%|██████████████████████████████████████████████████████████████████████▉                                       | 2575/3996 [3:11:49<1:40:10,  4.23s/it]
 64%|██████████████████████████████████████████████████████████████████████▉                                       | 2576/3996 [3:11:53<1:39:53,  4.22s/it]
 64%|██████████████████████████████████████████████████████████████████████▉                                       | 2577/3996 [3:11:58<1:45:06,  4.44s/it]
 65%|██████████████████████████████████████████████████████████████████████▉                                       | 2578/3996 [3:12:03<1:43:17,  4.37s/it]
 65%|██████████████████████████████████████████████████████████████████████▉                                       | 2579/3996 [3:12:07<1:41:56,  4.32s/it]
 65%|███████████████████████████████████████████████████████████████████████                                       | 2580/3996 [3:12:11<1:41:09,  4.29s/it]
 65%|███████████████████████████████████████████████████████████████████████                                       | 2581/3996 [3:12:15<1:41:52,  4.32s/it]
 65%|███████████████████████████████████████████████████████████████████████                                       | 2582/3996 [3:12:20<1:40:50,  4.28s/it]
 65%|███████████████████████████████████████████████████████████████████████                                       | 2583/3996 [3:12:24<1:40:14,  4.26s/it]
 65%|███████████████████████████████████████████████████████████████████████▏                                      | 2584/3996 [3:12:29<1:43:49,  4.41s/it]
 65%|███████████████████████████████████████████████████████████████████████▏                                      | 2585/3996 [3:12:33<1:42:13,  4.35s/it]
 65%|███████████████████████████████████████████████████████████████████████▏                                      | 2586/3996 [3:12:37<1:41:07,  4.30s/it]
 65%|███████████████████████████████████████████████████████████████████████▏                                      | 2587/3996 [3:12:41<1:40:19,  4.27s/it]
 65%|███████████████████████████████████████████████████████████████████████▏                                      | 2588/3996 [3:12:45<1:39:47,  4.25s/it]
 65%|███████████████████████████████████████████████████████████████████████▎                                      | 2589/3996 [3:12:50<1:39:17,  4.23s/it]
 65%|███████████████████████████████████████████████████████████████████████▎                                      | 2590/3996 [3:12:54<1:39:03,  4.23s/it]
 65%|███████████████████████████████████████████████████████████████████████▎                                      | 2591/3996 [3:12:58<1:42:35,  4.38s/it]
 65%|███████████████████████████████████████████████████████████████████████▎                                      | 2592/3996 [3:13:03<1:41:10,  4.32s/it]
 65%|███████████████████████████████████████████████████████████████████████▍                                      | 2593/3996 [3:13:07<1:40:11,  4.28s/it]
 65%|███████████████████████████████████████████████████████████████████████▍                                      | 2594/3996 [3:13:11<1:39:24,  4.25s/it]
 65%|███████████████████████████████████████████████████████████████████████▍                                      | 2595/3996 [3:13:15<1:38:58,  4.24s/it]
 65%|███████████████████████████████████████████████████████████████████████▍                                      | 2596/3996 [3:13:19<1:38:37,  4.23s/it]
 65%|███████████████████████████████████████████████████████████████████████▍                                      | 2597/3996 [3:13:24<1:38:17,  4.22s/it]
 65%|███████████████████████████████████████████████████████████████████████▌                                      | 2598/3996 [3:13:28<1:41:55,  4.37s/it]
 65%|███████████████████████████████████████████████████████████████████████▌                                      | 2599/3996 [3:13:33<1:40:40,  4.32s/it]
 65%|███████████████████████████████████████████████████████████████████████▌                                      | 2600/3996 [3:13:37<1:39:43,  4.29s/it]
                                                                                                                                                           
{'loss': 0.5017, 'grad_norm': 0.19957959651947021, 'learning_rate': 5.701683161910115e-05, 'ppl': 1.6515, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4244.94, 'total_tokens': 50147673, 'epoch': 1.95}
+
 65%|███████████████████████████████████████████████████████████████████████▌                                      | 2600/3996 [3:13:37<1:39:43,  4.29s/it]
 65%|███████████████████████████████████████████████████████████████████████▌                                      | 2601/3996 [3:13:41<1:38:56,  4.26s/it]
 65%|███████████████████████���███████████████████████████████████████████████▋                                      | 2602/3996 [3:13:45<1:38:32,  4.24s/it]
 65%|███████████████████████████████████████████████████████████████████████▋                                      | 2603/3996 [3:13:49<1:37:59,  4.22s/it]
 65%|███████████████████████████████████████████████████████████████████████▋                                      | 2604/3996 [3:13:54<1:37:45,  4.21s/it]
 65%|███████████████████████████████████████████████████████████████████████▋                                      | 2605/3996 [3:13:58<1:41:20,  4.37s/it]
 65%|███████████████████████████████████████████████████████████████████████▋                                      | 2606/3996 [3:14:02<1:40:06,  4.32s/it]
 65%|███████████████████████████████████████████████████████████████████████▊                                      | 2607/3996 [3:14:07<1:40:40,  4.35s/it]
 65%|███████████████████████████████████████████████████████████████████████▊                                      | 2608/3996 [3:14:11<1:39:32,  4.30s/it]
 65%|███████████████████████████████████████████████████████████████████████▊                                      | 2609/3996 [3:14:15<1:38:42,  4.27s/it]
 65%|███████████████████████████████████████████████████████████████████████▊                                      | 2610/3996 [3:14:20<1:42:46,  4.45s/it]
 65%|███████████████████████████████████████████████████████████████████████▊                                      | 2611/3996 [3:14:24<1:41:00,  4.38s/it]
 65%|███████████████████████████████████████████████████████████████████████▉                                      | 2612/3996 [3:14:29<1:43:33,  4.49s/it]
 65%|███████████████████████████████████████████████████████████████████████▉                                      | 2613/3996 [3:14:33<1:41:32,  4.41s/it]
 65%|███████████████████████████████████████████████████████████████████████▉                                      | 2614/3996 [3:14:38<1:40:04,  4.34s/it]
 65%|███████████████████████████████████████████████████████████████████████▉                                      | 2615/3996 [3:14:42<1:38:49,  4.29s/it]
 65%|████████████████████████████████████████████████████████████████████████                                      | 2616/3996 [3:14:46<1:38:02,  4.26s/it]
 65%|████████████████████████████████████████████████████████████████████████                                      | 2617/3996 [3:14:50<1:37:29,  4.24s/it]
 66%|████████████████████████████████████████████████████████████████████████                                      | 2618/3996 [3:14:54<1:37:06,  4.23s/it]
 66%|████████████████████████████████████████████████████████████████████████                                      | 2619/3996 [3:14:59<1:40:32,  4.38s/it]
 66%|████████████████████████████████████████████████████████████████████████                                      | 2620/3996 [3:15:03<1:39:14,  4.33s/it]
 66%|████████████████████████████████████████████████████████████████████████▏                                     | 2621/3996 [3:15:07<1:38:05,  4.28s/it]
 66%|████████████████████████████████████████████████████████████████████████▏                                     | 2622/3996 [3:15:12<1:37:24,  4.25s/it]
 66%|████████████████████████████████████████████████████████████████████████▏                                     | 2623/3996 [3:15:16<1:36:58,  4.24s/it]
 66%|████████████████████████████████████████████████████████████████████████▏                                     | 2624/3996 [3:15:20<1:36:34,  4.22s/it]
 66%|████████████████████████████████████████████████████████████████████████▎                                     | 2625/3996 [3:15:24<1:36:21,  4.22s/it]
                                                                                                                                                           
{'loss': 0.5024, 'grad_norm': 0.20284536480903625, 'learning_rate': 5.520550605921091e-05, 'ppl': 1.6527, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4205.45, 'total_tokens': 50589478, 'epoch': 1.97}
+
 66%|████████████████████████████████████████████████████████████████████████▎                                     | 2625/3996 [3:15:24<1:36:21,  4.22s/it]
 66%|████████████████████████████████████████████████████████████████████████▎                                     | 2626/3996 [3:15:29<1:39:43,  4.37s/it]
 66%|████████████████████████████████████████████████████████████████████████▎                                     | 2627/3996 [3:15:33<1:38:32,  4.32s/it]
 66%|████████████████████████████████████████████████████████████████████████▎                                     | 2628/3996 [3:15:37<1:37:33,  4.28s/it]
 66%|████████████████████████████████████████████████████████████████████████▎                                     | 2629/3996 [3:15:41<1:36:47,  4.25s/it]
 66%|████████████████████████████████████████████████████████████████████████▍                                     | 2630/3996 [3:15:46<1:36:27,  4.24s/it]
 66%|████████████████████████████████████████████████████████████████████████▍                                     | 2631/3996 [3:15:50<1:39:14,  4.36s/it]
 66%|████████████████████████████████████████████████████████████████████████▍                                     | 2632/3996 [3:15:55<1:38:02,  4.31s/it]
 66%|████████████████████████████████████████████████████████████████████████▍                                     | 2633/3996 [3:15:59<1:40:45,  4.44s/it]
 66%|████████████████████████████████████████████████████████████████████████▌                                     | 2634/3996 [3:16:03<1:39:04,  4.36s/it]
 66%|████████████████████████████████████████████████████████████████████████▌                                     | 2635/3996 [3:16:08<1:37:48,  4.31s/it]
 66%|████████████████████████████████████████████████████████████████████████▌                                     | 2636/3996 [3:16:12<1:36:47,  4.27s/it]
 66%|████████████████████████████████████████████████████████████████████████▌                                     | 2637/3996 [3:16:16<1:36:16,  4.25s/it]
 66%|████████████████████████████████████████████████████████████████████████▌                                     | 2638/3996 [3:16:21<1:38:42,  4.36s/it]
 66%|████████████████████████████████████████████████████████████████████████▋                                     | 2639/3996 [3:16:25<1:37:30,  4.31s/it]
 66%|████████████████████████████████████████████████████████████████████████▋                                     | 2640/3996 [3:16:30<1:44:53,  4.64s/it]
 66%|████████████████████████████████████████████████████████████████████████▋                                     | 2641/3996 [3:16:34<1:41:49,  4.51s/it]
 66%|████████████████████████████████████████████████████████████████████████▋                                     | 2642/3996 [3:16:39<1:39:32,  4.41s/it]
 66%|████████████████████████████████████████████████████████████████████████▊                                     | 2643/3996 [3:16:43<1:37:53,  4.34s/it]
 66%|████████████████████████████████████████████████████████████████████████▊                                     | 2644/3996 [3:16:47<1:36:47,  4.30s/it]
 66%|████████████████████████████████████████████████████████████████████████▊                                     | 2645/3996 [3:16:51<1:35:55,  4.26s/it]
 66%|████████████████████████████████████████████████████████████████████████▊                                     | 2646/3996 [3:16:55<1:35:22,  4.24s/it]
 66%|████████████████████████████████████████████████████████████████████████▊                                     | 2647/3996 [3:17:00<1:38:28,  4.38s/it]
 66%|████████████████████████████████████████████████████████████████████████▉                                     | 2648/3996 [3:17:04<1:37:09,  4.32s/it]
 66%|████████████████████████████████████████████████████████████████████████▉                                     | 2649/3996 [3:17:08<1:36:13,  4.29s/it]
 66%|████████████████████████████████████████████████████████████████████████▉                                     | 2650/3996 [3:17:13<1:35:22,  4.25s/it]
                                                                                                                                                           
{'loss': 0.4967, 'grad_norm': 0.2044789344072342, 'learning_rate': 5.34123838691753e-05, 'ppl': 1.6433, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4204.9, 'total_tokens': 51027800, 'epoch': 1.99}
+
 66%|████████████████████████████████████████████████████████████████████████▉                                     | 2650/3996 [3:17:13<1:35:22,  4.25s/it]
 66%|████████████████████████████████████████████████████████████████████████▉                                     | 2651/3996 [3:17:17<1:34:59,  4.24s/it]
 66%|█████████████████████████████████████████████████████████████████████████                                     | 2652/3996 [3:17:21<1:34:34,  4.22s/it]
 66%|█████████████████████████████████████████████████████████████████████████                                     | 2653/3996 [3:17:25<1:34:13,  4.21s/it]
 66%|█████████████████████████████████████████████████████████████████████████                                     | 2654/3996 [3:17:30<1:37:37,  4.36s/it]
 66%|█████████████████████████████████████████████████████████████████████████                                     | 2655/3996 [3:17:34<1:36:25,  4.31s/it]
 66%|█████████████████████████████████████████████████████████████████████████                                     | 2656/3996 [3:17:38<1:35:20,  4.27s/it]
 66%|█████████████████████████████████████████████████████████████████████████▏                                    | 2657/3996 [3:17:42<1:34:41,  4.24s/it]
 67%|█████████████████████████████████████████████████████████████████████████▏                                    | 2658/3996 [3:17:47<1:34:14,  4.23s/it]
 67%|█████████████████████████████████████████████████████████████████████████▏                                    | 2659/3996 [3:17:51<1:33:52,  4.21s/it]
 67%|█████████████████████████████████████████████████████████████████████████▏                                    | 2660/3996 [3:17:55<1:33:35,  4.20s/it]
 67%|█████████████████████████████████████████████████████████████████████████▎                                    | 2661/3996 [3:18:00<1:36:53,  4.35s/it]
 67%|█████████████████████████████████████████████████████████████████████████▎                                    | 2662/3996 [3:18:04<1:35:42,  4.30s/it]
 67%|█████████████████████████████████████████████████████████████████████████▎                                    | 2663/3996 [3:18:08<1:34:49,  4.27s/it]
 67%|█████████████████████████████████████████████████████████████████████████▎                                    | 2664/3996 [3:18:12<1:34:06,  4.24s/it]
 67%|████████████���████████████████████████████████████████████████████████████▎                                    | 2665/3996 [3:18:16<1:33:40,  4.22s/it]
 67%|█████████████████████████████████████████████████████████████████████████▍                                    | 2666/3996 [3:18:21<1:33:41,  4.23s/it]
 67%|█████████████████████████████████████████████████████████████████████████▍                                    | 2667/3996 [3:18:26<1:43:59,  4.69s/it]
 67%|█████████████████████████████████████████████████████████████████████████▍                                    | 2668/3996 [3:18:32<1:46:26,  4.81s/it]
 67%|█████████████████████████████████████████████████████████████████████████▍                                    | 2669/3996 [3:18:36<1:42:33,  4.64s/it]
 67%|█████████████████████████████████████████████████████████████████████████▍                                    | 2670/3996 [3:18:40<1:39:31,  4.50s/it]
 67%|█████████████████████████████████████████████████████████████████████████▌                                    | 2671/3996 [3:18:44<1:37:16,  4.41s/it]
 67%|█████████████████████████████████████████████████████████████████████████▌                                    | 2672/3996 [3:18:48<1:35:51,  4.34s/it]
 67%|█████████████████████████████████████████████████████████████████████████▌                                    | 2673/3996 [3:18:53<1:34:51,  4.30s/it]
 67%|█████████████████████████████████████████████████████████████████████████▌                                    | 2674/3996 [3:18:57<1:34:02,  4.27s/it]
 67%|█████████████████████████████████████████████████████████████████████████▋                                    | 2675/3996 [3:19:01<1:36:58,  4.40s/it]
                                                                                                                                                           
{'loss': 0.4862, 'grad_norm': 0.2125943899154663, 'learning_rate': 5.163819372931979e-05, 'ppl': 1.6261, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3745.54, 'total_tokens': 51469941, 'epoch': 2.01}
+
 67%|█████████████████████████████████████████████████████████████████████████▋                                    | 2675/3996 [3:19:01<1:36:58,  4.40s/it]
 67%|█████████████████████████████████████████████████████████████████████████▋                                    | 2676/3996 [3:19:06<1:35:32,  4.34s/it]
 67%|█████████████████████████████████████████████████████████████████████████▋                                    | 2677/3996 [3:19:10<1:34:26,  4.30s/it]
 67%|█████████████████████████████████████████████████████████████████████████▋                                    | 2678/3996 [3:19:14<1:33:37,  4.26s/it]
 67%|█████████████████████████████████████████████████████████████████████████▋                                    | 2679/3996 [3:19:18<1:33:05,  4.24s/it]
 67%|█████████████████████████████████████████████████████████████████████████▊                                    | 2680/3996 [3:19:23<1:34:09,  4.29s/it]
 67%|█████████████████████████████████████████████████████████████████████████▊                                    | 2681/3996 [3:19:27<1:33:22,  4.26s/it]
 67%|█████████████████████████████████████████████████████████████████████████▊                                    | 2682/3996 [3:19:32<1:36:26,  4.40s/it]
 67%|█████████████████████████████████████████████████████████████████████████▊                                    | 2683/3996 [3:19:36<1:37:39,  4.46s/it]
 67%|█████████████████████████████████████████████████████████████████████████▉                                    | 2684/3996 [3:19:40<1:36:05,  4.39s/it]
 67%|█████████████████████████████████████████████████████████████████████████▉                                    | 2685/3996 [3:19:45<1:34:43,  4.34s/it]
 67%|█████████████████████████████████████████████████████████████████████████▉                                    | 2686/3996 [3:19:49<1:33:50,  4.30s/it]
 67%|█████████████████████████████████████████████████████████████████████████▉                                    | 2687/3996 [3:19:53<1:33:13,  4.27s/it]
 67%|█████████████████████████████████████████████████████████████████████████▉                                    | 2688/3996 [3:19:57<1:32:43,  4.25s/it]
 67%|██████████████████████████████████████████████████████████████████████████                                    | 2689/3996 [3:20:02<1:35:49,  4.40s/it]
 67%|██████████████████████████████████████████████████████████████████████████                                    | 2690/3996 [3:20:06<1:34:28,  4.34s/it]
 67%|██████████████████████████████████████████████████████████████████████████                                    | 2691/3996 [3:20:10<1:33:32,  4.30s/it]
 67%|██████████████████████████████████████████████████████████████████████████                                    | 2692/3996 [3:20:15<1:32:42,  4.27s/it]
 67%|██████████████████████████████████████████████████████████████████████████▏                                   | 2693/3996 [3:20:19<1:32:12,  4.25s/it]
 67%|██████████████████████████████████████████████████████████████████████████▏                                   | 2694/3996 [3:20:23<1:31:49,  4.23s/it]
 67%|██████████████████████████████████████████████████████████████████████████▏                                   | 2695/3996 [3:20:27<1:33:34,  4.32s/it]
 67%|██████████████████████████████████████████████████████████████████████████▏                                   | 2696/3996 [3:20:32<1:36:20,  4.45s/it]
 67%|██████████████████████████████████████████████████████████████████████████▏                                   | 2697/3996 [3:20:36<1:34:47,  4.38s/it]
 68%|██████████████████████████████████████████████████████████████████████████▎                                   | 2698/3996 [3:20:41<1:33:35,  4.33s/it]
 68%|██████████████████████████████████████████████████████████████████████████▎                                   | 2699/3996 [3:20:45<1:32:39,  4.29s/it]
 68%|██████████████████████████████████████████████████████████████████████████▎                                   | 2700/3996 [3:20:49<1:32:09,  4.27s/it]
                                                                                                                                                           
{'loss': 0.4782, 'grad_norm': 0.2312517911195755, 'learning_rate': 4.9883656626454724e-05, 'ppl': 1.6132, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4275.5, 'total_tokens': 51921057, 'epoch': 2.03}
+
 68%|██████████████████████████████████████████████████████████████████████████▎                                   | 2700/3996 [3:20:49<1:32:09,  4.27s/it]
 68%|██████████████████████████████████████████████████████████████████████████▎                                   | 2701/3996 [3:20:53<1:31:36,  4.24s/it]
 68%|██████████████████████████████████████████████████████████████████████████▍                                   | 2702/3996 [3:20:57<1:31:15,  4.23s/it]
 68%|██████████████████████████████████████████████████████████████████████████▍                                   | 2703/3996 [3:21:02<1:34:32,  4.39s/it]
 68%|██████████████████████████████████████████████████████████████████████████▍                                   | 2704/3996 [3:21:06<1:33:21,  4.34s/it]
 68%|██████████████████████████████████████████████████████████████████████████▍                                   | 2705/3996 [3:21:11<1:32:18,  4.29s/it]
 68%|██████████████████████████████████████████████████████████████████████████▍                                   | 2706/3996 [3:21:15<1:31:34,  4.26s/it]
 68%|██████████████████████████████████████████████████████████████████████████▌                                   | 2707/3996 [3:21:19<1:31:35,  4.26s/it]
 68%|██████████████████████████████████████████████████████████████████████████▌                                   | 2708/3996 [3:21:23<1:31:09,  4.25s/it]
 68%|██████████████████████████████████████████████████████████████████████████▌                                   | 2709/3996 [3:21:27<1:30:50,  4.24s/it]
 68%|███████████████████████████████████████████████████████��██████████████████▌                                   | 2710/3996 [3:21:32<1:34:06,  4.39s/it]
 68%|██████████████████████████████████████████████████████████████████████████▋                                   | 2711/3996 [3:21:36<1:32:57,  4.34s/it]
 68%|██████████████████████████████████████████████████████████████████████████▋                                   | 2712/3996 [3:21:41<1:32:02,  4.30s/it]
 68%|██████████████████████████████████████████████████████████████████████████▋                                   | 2713/3996 [3:21:45<1:31:20,  4.27s/it]
 68%|██████████████████████████████████████████████████████████████████████████▋                                   | 2714/3996 [3:21:49<1:30:52,  4.25s/it]
 68%|██████████████████████████████████████████████████████████████████████████▋                                   | 2715/3996 [3:21:53<1:30:29,  4.24s/it]
 68%|██████████████████████████████████████████████████████████████████████████▊                                   | 2716/3996 [3:21:57<1:30:09,  4.23s/it]
 68%|██████████████████████████████████████████████████████████████████████████▊                                   | 2717/3996 [3:22:02<1:33:22,  4.38s/it]
 68%|██████████████████████████████████████████████████████████████████████████▊                                   | 2718/3996 [3:22:06<1:32:10,  4.33s/it]
 68%|██████████████████████████████████████████████████████████████████████████▊                                   | 2719/3996 [3:22:11<1:31:17,  4.29s/it]
 68%|██████████████████████████████████████████████████████████████████████████▊                                   | 2720/3996 [3:22:15<1:30:33,  4.26s/it]
 68%|██████████████████████████████████████████████████████████████████████████▉                                   | 2721/3996 [3:22:19<1:31:11,  4.29s/it]
 68%|██████████████████████████████████████████████████████████████████████████▉                                   | 2722/3996 [3:22:23<1:30:33,  4.26s/it]
 68%|██████████████████████████████████████████████████████████████████████████▉                                   | 2723/3996 [3:22:28<1:30:05,  4.25s/it]
 68%|██████████████████████████████████████████████████████████████████████████▉                                   | 2724/3996 [3:22:32<1:33:11,  4.40s/it]
 68%|███████████████████████████████████████████████████████████████████████████                                   | 2725/3996 [3:22:37<1:31:56,  4.34s/it]
                                                                                                                                                           
{'loss': 0.4717, 'grad_norm': 0.19745635986328125, 'learning_rate': 4.81494855608843e-05, 'ppl': 1.6027, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4290.88, 'total_tokens': 52372623, 'epoch': 2.04}
+
 68%|███████████████████████████████████████████████████████████████████████████                                   | 2725/3996 [3:22:37<1:31:56,  4.34s/it]
 68%|███████████████████████████████████████████████████████████████████████████                                   | 2726/3996 [3:22:41<1:30:58,  4.30s/it]
 68%|███████████████████████████████████████████████████████████████████████████                                   | 2727/3996 [3:22:45<1:30:15,  4.27s/it]
 68%|███████████████████████████████████████████████████████████████████████████                                   | 2728/3996 [3:22:49<1:29:48,  4.25s/it]
 68%|███████████████████████████████████████████████████████████████████████████                                   | 2729/3996 [3:22:53<1:29:22,  4.23s/it]
 68%|███████████████████████████████████████████████████████████████████████████▏                                  | 2730/3996 [3:22:58<1:29:06,  4.22s/it]
 68%|███████████████████████████████████████████████████████████████████████████▏                                  | 2731/3996 [3:23:02<1:32:17,  4.38s/it]
 68%|███████████████████████████████████████████████████████████████████████████▏                                  | 2732/3996 [3:23:06<1:31:13,  4.33s/it]
 68%|███████████████████████████████████████████████████████████████████████████▏                                  | 2733/3996 [3:23:11<1:30:15,  4.29s/it]
 68%|███████████████████████████████████████████████████████████████████████████▎                                  | 2734/3996 [3:23:15<1:29:30,  4.26s/it]
 68%|███████████████████████████████████████████████████████████████████████████▎                                  | 2735/3996 [3:23:19<1:29:07,  4.24s/it]
 68%|███████████████████████████████████████████████████████████████████████████▎                                  | 2736/3996 [3:23:23<1:28:45,  4.23s/it]
 68%|███████████████████████████████████████████████████████████████████████████▎                                  | 2737/3996 [3:23:27<1:28:35,  4.22s/it]
 69%|███████████████████████████████████████████████████████████████████████████▎                                  | 2738/3996 [3:23:33<1:34:51,  4.52s/it]
 69%|███████████████████████████████████████████████████████████████████████████▍                                  | 2739/3996 [3:23:37<1:32:49,  4.43s/it]
 69%|███████████████████████████████████████████████████████████████████████████▍                                  | 2740/3996 [3:23:41<1:31:17,  4.36s/it]
 69%|██████████████████████████████████████████████��████████████████████████████▍                                  | 2741/3996 [3:23:45<1:30:11,  4.31s/it]
 69%|███████████████████████████████████████████████████████████████████████████▍                                  | 2742/3996 [3:23:50<1:29:31,  4.28s/it]
 69%|███████████████████████████████████████████████████████████████████████████▌                                  | 2743/3996 [3:23:54<1:28:52,  4.26s/it]
 69%|███████████████████████████████████████████████████████████████████████████▌                                  | 2744/3996 [3:23:58<1:28:26,  4.24s/it]
 69%|███████████████████████████████████████████████████████████████████████████▌                                  | 2745/3996 [3:24:03<1:31:30,  4.39s/it]
 69%|███████████████████████████████████████████████████████████████████████████▌                                  | 2746/3996 [3:24:07<1:30:21,  4.34s/it]
 69%|███████████████████████████████████████████████████████████████████████████▌                                  | 2747/3996 [3:24:11<1:29:21,  4.29s/it]
 69%|███████████████████████████████████████████████████████████████████████████▋                                  | 2748/3996 [3:24:15<1:28:33,  4.26s/it]
 69%|███████████████████████████████████████████████████████████████████████████▋                                  | 2749/3996 [3:24:19<1:28:15,  4.25s/it]
 69%|███████████████████████████████████████████████████████████████████████████▋                                  | 2750/3996 [3:24:24<1:27:53,  4.23s/it]
                                                                                                                                                           
{'loss': 0.4817, 'grad_norm': 0.22817276418209076, 'learning_rate': 4.643638525666095e-05, 'ppl': 1.6188, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4292.31, 'total_tokens': 52823263, 'epoch': 2.06}
+
 69%|███████████████████████████████████████████████████████████████████████████▋                                  | 2750/3996 [3:24:24<1:27:53,  4.23s/it]
 69%|███████████████████████████████████████████████████████████████████████████▋                                  | 2751/3996 [3:24:28<1:27:37,  4.22s/it]
 69%|███████████████████████████████████████████████████████████████████████████▊                                  | 2752/3996 [3:24:33<1:30:43,  4.38s/it]
 69%|███████████████████████████████████████████████████████████████████████████▊                                  | 2753/3996 [3:24:37<1:29:35,  4.32s/it]
 69%|███████████████████████████████████████████████████████████████████████████▊                                  | 2754/3996 [3:24:41<1:28:44,  4.29s/it]
 69%|███████████████████████████████████████████████████████████████████████████▊                                  | 2755/3996 [3:24:45<1:28:06,  4.26s/it]
 69%|███████████████████████████████████████████████████████████████████████████▊                                  | 2756/3996 [3:24:49<1:27:46,  4.25s/it]
 69%|███████████████████████████████████████████████████████████████████████████▉                                  | 2757/3996 [3:24:54<1:27:25,  4.23s/it]
 69%|███████████████████████████████████████████████████████████████████████████▉                                  | 2758/3996 [3:24:58<1:27:04,  4.22s/it]
 69%|███████████████████████████████████████████████████████████████████████████▉                                  | 2759/3996 [3:25:03<1:30:17,  4.38s/it]
 69%|███████████████████████████████████████████████████████████████████████████▉                                  | 2760/3996 [3:25:07<1:29:10,  4.33s/it]
 69%|████████████████████████████████████████████████████████████████████████████                                  | 2761/3996 [3:25:11<1:28:18,  4.29s/it]
 69%|████████████████████████████████████████████████████████████████████████████                                  | 2762/3996 [3:25:15<1:27:34,  4.26s/it]
 69%|████████████████████████████████████████████████████████████████████████████                                  | 2763/3996 [3:25:19<1:27:11,  4.24s/it]
 69%|████████████████████████████████████████████████████████████████████████████                                  | 2764/3996 [3:25:24<1:26:49,  4.23s/it]
 69%|████████████████████████████████████████████████████████████████████████████                                  | 2765/3996 [3:25:28<1:26:33,  4.22s/it]
 69%|████████████████████████████████████████████████████████████████████████████▏                                 | 2766/3996 [3:25:32<1:29:39,  4.37s/it]
 69%|████████████████████████████████████████████████████████████████████████████▏                                 | 2767/3996 [3:25:37<1:28:34,  4.32s/it]
 69%|████████████████████████████████████████████████████████████████████████████▏                                 | 2768/3996 [3:25:41<1:27:40,  4.28s/it]
 69%|████████████████████████████████████████████████████████████████████████████▏                                 | 2769/3996 [3:25:45<1:27:04,  4.26s/it]
 69%|████████████████████████████████████████████████████████████████████████████▎                                 | 2770/3996 [3:25:49<1:26:41,  4.24s/it]
 69%|████████████████████████████████████████████████████████████████████████████▎                                 | 2771/3996 [3:25:53<1:26:13,  4.22s/it]
 69%|███████████████████���████████████████████████████████████████████████████████▎                                 | 2772/3996 [3:25:58<1:25:59,  4.22s/it]
 69%|████████████████████████████████████████████████████████████████████████████▎                                 | 2773/3996 [3:26:02<1:29:10,  4.38s/it]
 69%|████████████████████████████████████████████████████████████████████████████▎                                 | 2774/3996 [3:26:07<1:28:05,  4.33s/it]
 69%|████████████████████████████████████████████████████████████████████████████▍                                 | 2775/3996 [3:26:11<1:27:13,  4.29s/it]
                                                                                                                                                           
{'loss': 0.4774, 'grad_norm': 0.20878754556179047, 'learning_rate': 4.4745051875203134e-05, 'ppl': 1.6119, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4287.12, 'total_tokens': 53272669, 'epoch': 2.08}
+
 69%|████████████████████████████████████████████████████████████████████████████▍                                 | 2775/3996 [3:26:11<1:27:13,  4.29s/it]
 69%|████████████████████████████████████████████████████████████████████████████▍                                 | 2776/3996 [3:26:15<1:26:32,  4.26s/it]
 69%|████████████████████████████████████████████████████████████████████████████▍                                 | 2777/3996 [3:26:19<1:26:09,  4.24s/it]
 70%|████████████████████████████████████████████████████████████████████████████▍                                 | 2778/3996 [3:26:23<1:25:48,  4.23s/it]
 70%|████████████████████████████████████████████████████████████████████████████▍                                 | 2779/3996 [3:26:28<1:25:35,  4.22s/it]
 70%|████████████████████████████████████████████████████████████████████████████▌                                 | 2780/3996 [3:26:32<1:28:39,  4.37s/it]
 70%|████████████████████████████████████████████████████████████████████████████▌                                 | 2781/3996 [3:26:37<1:27:33,  4.32s/it]
 70%|████████████████████████████████████████████████████████████████████████████▌                                 | 2782/3996 [3:26:41<1:26:42,  4.29s/it]
 70%|████████████████████████████████████████████████████████████████████████████▌                                 | 2783/3996 [3:26:45<1:26:03,  4.26s/it]
 70%|████████████████████████████████████████████████████████████████████████████▋                                 | 2784/3996 [3:26:49<1:25:42,  4.24s/it]
 70%|████████████████████████████████████████████████████████████████████████████▋                                 | 2785/3996 [3:26:53<1:25:20,  4.23s/it]
 70%|████████████████████████���███████████████████████████████████████████████████▋                                 | 2786/3996 [3:26:58<1:25:05,  4.22s/it]
 70%|████████████████████████████████████████████████████████████████████████████▋                                 | 2787/3996 [3:27:02<1:28:13,  4.38s/it]
 70%|████████████████████████████████████████████████████████████████████████████▋                                 | 2788/3996 [3:27:06<1:27:04,  4.32s/it]
 70%|████████████████████████████████████████████████████████████████████████████▊                                 | 2789/3996 [3:27:11<1:26:15,  4.29s/it]
 70%|████████████████████████████████████████████████████████████████████████████▊                                 | 2790/3996 [3:27:15<1:25:37,  4.26s/it]
 70%|████████████████████████████████████████████████████████████████████████████▊                                 | 2791/3996 [3:27:19<1:25:15,  4.25s/it]
 70%|████████████████████████████████████████████████████████████████████████████▊                                 | 2792/3996 [3:27:23<1:24:57,  4.23s/it]
 70%|████████████████████████████████████████████████████████████████████████████▉                                 | 2793/3996 [3:27:28<1:27:04,  4.34s/it]
 70%|████████████████████████████████████████████████████████████████████████████▉                                 | 2794/3996 [3:27:33<1:29:23,  4.46s/it]
 70%|████████████████████████████████████████████████████████████████████████████▉                                 | 2795/3996 [3:27:37<1:27:44,  4.38s/it]
 70%|████████████████████████████████████████████████████████████████████████████▉                                 | 2796/3996 [3:27:41<1:26:33,  4.33s/it]
 70%|████████████████████████████████████████████████████████████████████████████▉                                 | 2797/3996 [3:27:45<1:25:39,  4.29s/it]
 70%|█████████████████████████████████████████████████████████████████████████████                                 | 2798/3996 [3:27:49<1:25:10,  4.27s/it]
 70%|█████████████████████████████████████████████████████████████████████████████                                 | 2799/3996 [3:27:54<1:24:38,  4.24s/it]
 70%|█████████████████████████████████████████████████████████████████████████████                                 | 2800/3996 [3:27:58<1:24:19,  4.23s/it]
                                                                                                                                                           
{'loss': 0.4824, 'grad_norm': 0.18676196038722992, 'learning_rate': 4.307617273239226e-05, 'ppl': 1.62, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4304.14, 'total_tokens': 53724750, 'epoch': 2.1}
+
 70%|██████████████████████████████��██████████████████████████████████████████████                                 | 2800/3996 [3:27:58<1:24:19,  4.23s/it]
 70%|█████████████████████████████████████████████████████████████████████████████                                 | 2801/3996 [3:28:03<1:27:17,  4.38s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▏                                | 2802/3996 [3:28:07<1:26:08,  4.33s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▏                                | 2803/3996 [3:28:11<1:25:15,  4.29s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▏                                | 2804/3996 [3:28:15<1:24:37,  4.26s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▏                                | 2805/3996 [3:28:19<1:24:14,  4.24s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▏                                | 2806/3996 [3:28:24<1:24:27,  4.26s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▎                                | 2807/3996 [3:28:28<1:24:06,  4.24s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▎                                | 2808/3996 [3:28:33<1:27:00,  4.39s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▎                                | 2809/3996 [3:28:37<1:25:49,  4.34s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▎                                | 2810/3996 [3:28:41<1:24:53,  4.29s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▍                                | 2811/3996 [3:28:45<1:24:06,  4.26s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▍                                | 2812/3996 [3:28:49<1:23:47,  4.25s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▍                                | 2813/3996 [3:28:54<1:23:20,  4.23s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▍                                | 2814/3996 [3:28:58<1:23:06,  4.22s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▍                                | 2815/3996 [3:29:03<1:26:08,  4.38s/it]
 70%|██████████████████████████████████████████████████████████████████████████��██▌                                | 2816/3996 [3:29:07<1:25:00,  4.32s/it]
 70%|█████████████████████████████████████████████████████████████████████████████▌                                | 2817/3996 [3:29:11<1:24:13,  4.29s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▌                                | 2818/3996 [3:29:15<1:23:31,  4.25s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▌                                | 2819/3996 [3:29:19<1:24:10,  4.29s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▋                                | 2820/3996 [3:29:24<1:23:34,  4.26s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▋                                | 2821/3996 [3:29:28<1:23:11,  4.25s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▋                                | 2822/3996 [3:29:33<1:25:59,  4.39s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▋                                | 2823/3996 [3:29:37<1:24:46,  4.34s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▋                                | 2824/3996 [3:29:41<1:24:12,  4.31s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▊                                | 2825/3996 [3:29:45<1:23:23,  4.27s/it]
                                                                                                                                                           
{'loss': 0.4701, 'grad_norm': 0.20670537650585175, 'learning_rate': 4.1430426019264924e-05, 'ppl': 1.6002, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4283.76, 'total_tokens': 54172957, 'epoch': 2.12}
+
 71%|█████████████████████████████████████████████████████████████████████████████▊                                | 2825/3996 [3:29:45<1:23:23,  4.27s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▊                                | 2826/3996 [3:29:49<1:22:58,  4.26s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▊                                | 2827/3996 [3:29:54<1:22:29,  4.23s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▊                                | 2828/3996 [3:29:58<1:22:12,  4.22s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▉                                | 2829/3996 [3:30:03<1:25:52,  4.42s/it]
 71%|█████████████████████████████████████████████████████████████████████��███████▉                                | 2830/3996 [3:30:07<1:24:37,  4.35s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▉                                | 2831/3996 [3:30:11<1:23:40,  4.31s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▉                                | 2832/3996 [3:30:15<1:22:58,  4.28s/it]
 71%|█████████████████████████████████████████████████████████████████████████████▉                                | 2833/3996 [3:30:20<1:22:28,  4.26s/it]
 71%|██████████████████████████████████████████████████████████████████████████████                                | 2834/3996 [3:30:24<1:22:07,  4.24s/it]
 71%|██████████████████████████████████████████████████████████████████████████████                                | 2835/3996 [3:30:28<1:21:49,  4.23s/it]
 71%|██████████████████████████████████████████████████████████████████████████████                                | 2836/3996 [3:30:33<1:24:49,  4.39s/it]
 71%|██████████████████████████████████████████████████████████████████████████████                                | 2837/3996 [3:30:37<1:23:43,  4.33s/it]
 71%|██████████████████████████████████████████████████████████████████████████████                                | 2838/3996 [3:30:41<1:22:51,  4.29s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▏                               | 2839/3996 [3:30:45<1:22:10,  4.26s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▏                               | 2840/3996 [3:30:50<1:21:48,  4.25s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▏                               | 2841/3996 [3:30:54<1:21:29,  4.23s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▏                               | 2842/3996 [3:30:58<1:21:15,  4.23s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▎                               | 2843/3996 [3:31:03<1:24:13,  4.38s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▎                               | 2844/3996 [3:31:07<1:23:06,  4.33s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▎                               | 2845/3996 [3:31:11<1:22:18,  4.29s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▎                               | 2846/3996 [3:31:15<1:21:40,  4.26s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▎                               | 2847/3996 [3:31:20<1:22:46,  4.32s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▍                               | 2848/3996 [3:31:24<1:22:03,  4.29s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▍                               | 2849/3996 [3:31:28<1:21:31,  4.26s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▍                               | 2850/3996 [3:31:33<1:24:33,  4.43s/it]
                                                                                                                                                           
{'loss': 0.4772, 'grad_norm': 0.21445906162261963, 'learning_rate': 3.980848052641286e-05, 'ppl': 1.6116, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3768.93, 'total_tokens': 54625827, 'epoch': 2.14}
+
 71%|██████████████████████████████████████████████████████████████████████████████▍                               | 2850/3996 [3:31:33<1:24:33,  4.43s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▍                               | 2851/3996 [3:31:37<1:23:17,  4.36s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▌                               | 2852/3996 [3:31:41<1:22:16,  4.32s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▌                               | 2853/3996 [3:31:46<1:21:28,  4.28s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▌                               | 2854/3996 [3:31:50<1:20:56,  4.25s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▌                               | 2855/3996 [3:31:54<1:20:36,  4.24s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▌                               | 2856/3996 [3:31:58<1:20:21,  4.23s/it]
 71%|██████████████████████████████████████████████████████████████████████████████▋                               | 2857/3996 [3:32:03<1:23:17,  4.39s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▋                               | 2858/3996 [3:32:07<1:22:15,  4.34s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▋                               | 2859/3996 [3:32:11<1:21:19,  4.29s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▋                               | 2860/3996 [3:32:16<1:20:43,  4.26s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▊                               | 2861/3996 [3:32:20<1:20:19,  4.25s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▊                               | 2862/3996 [3:32:24<1:20:02,  4.23s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▊                               | 2863/3996 [3:32:28<1:19:46,  4.22s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▊                               | 2864/3996 [3:32:33<1:22:44,  4.39s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▊                               | 2865/3996 [3:32:37<1:21:38,  4.33s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▉                               | 2866/3996 [3:32:41<1:20:45,  4.29s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▉                               | 2867/3996 [3:32:45<1:20:06,  4.26s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▉                               | 2868/3996 [3:32:50<1:19:44,  4.24s/it]
 72%|██████████████████████████████████████████████████████████████████████████████▉                               | 2869/3996 [3:32:54<1:19:27,  4.23s/it]
 72%|███████████████████████████████████████████████████████████████████████████████                               | 2870/3996 [3:32:58<1:19:09,  4.22s/it]
 72%|███████████████████████████████████████████████████████████████████████████████                               | 2871/3996 [3:33:03<1:22:05,  4.38s/it]
 72%|███████████████████████████████████████████████████████████████████████████████                               | 2872/3996 [3:33:07<1:21:07,  4.33s/it]
 72%|███████████████████████████████████████████████████████████████████████████████                               | 2873/3996 [3:33:11<1:20:19,  4.29s/it]
 72%|███████████████████████████████████████████████████████████████████████████████                               | 2874/3996 [3:33:15<1:19:44,  4.26s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▏                              | 2875/3996 [3:33:20<1:19:19,  4.25s/it]
                                                                                                                                                           
{'loss': 0.471, 'grad_norm': 0.21021129190921783, 'learning_rate': 3.8210995372202896e-05, 'ppl': 1.6016, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4286.55, 'total_tokens': 55076031, 'epoch': 2.16}
+
 72%|███████████████████████████████████████████████████████████████████████████████▏                              | 2875/3996 [3:33:20<1:19:19,  4.25s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▏                              | 2876/3996 [3:33:24<1:19:02,  4.23s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▏                              | 2877/3996 [3:33:28<1:18:49,  4.23s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▏                              | 2878/3996 [3:33:33<1:21:41,  4.38s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▎                              | 2879/3996 [3:33:37<1:20:38,  4.33s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▎                              | 2880/3996 [3:33:41<1:19:53,  4.30s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▎                              | 2881/3996 [3:33:45<1:19:14,  4.26s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▎                              | 2882/3996 [3:33:50<1:18:50,  4.25s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▎                              | 2883/3996 [3:33:54<1:20:11,  4.32s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▍                              | 2884/3996 [3:33:58<1:19:21,  4.28s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▍                              | 2885/3996 [3:34:03<1:21:52,  4.42s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▍                              | 2886/3996 [3:34:07<1:20:46,  4.37s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▍                              | 2887/3996 [3:34:12<1:19:47,  4.32s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▍                              | 2888/3996 [3:34:16<1:19:03,  4.28s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▌                              | 2889/3996 [3:34:20<1:18:36,  4.26s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▌                              | 2890/3996 [3:34:24<1:18:12,  4.24s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▌                              | 2891/3996 [3:34:28<1:17:58,  4.23s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▌                              | 2892/3996 [3:34:33<1:20:39,  4.38s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▋                              | 2893/3996 [3:34:37<1:19:34,  4.33s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▋                              | 2894/3996 [3:34:41<1:18:48,  4.29s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▋                              | 2895/3996 [3:34:46<1:18:14,  4.26s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▋                              | 2896/3996 [3:34:50<1:17:54,  4.25s/it]
 72%|███████████████████████████████████████████████████████████████████████████████▋                              | 2897/3996 [3:34:54<1:17:32,  4.23s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▊                              | 2898/3996 [3:34:58<1:17:12,  4.22s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▊                              | 2899/3996 [3:35:03<1:20:02,  4.38s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▊                              | 2900/3996 [3:35:07<1:19:03,  4.33s/it]
                                                                                                                                                           
{'loss': 0.4722, 'grad_norm': 0.23069453239440918, 'learning_rate': 3.663861973492776e-05, 'ppl': 1.6035, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4291.53, 'total_tokens': 55527864, 'epoch': 2.18}
+
 73%|███████████████████████████████████████████████████████████████████████████████▊                              | 2900/3996 [3:35:07<1:19:03,  4.33s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▊                              | 2901/3996 [3:35:12<1:19:24,  4.35s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▉                              | 2902/3996 [3:35:16<1:18:26,  4.30s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▉                              | 2903/3996 [3:35:20<1:17:51,  4.27s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▉                              | 2904/3996 [3:35:24<1:17:17,  4.25s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▉                              | 2905/3996 [3:35:28<1:17:02,  4.24s/it]
 73%|███████████████████████████████████████████████████████████████████████████████▉                              | 2906/3996 [3:35:33<1:19:43,  4.39s/it]
 73%|████████████████████████████████████████████████████████████████████████████████                              | 2907/3996 [3:35:37<1:18:39,  4.33s/it]
 73%|████████████████████████████████████████████████████████████████████████████████                              | 2908/3996 [3:35:42<1:17:52,  4.29s/it]
 73%|████████████████████████████████████████████████████████████████████████████████                              | 2909/3996 [3:35:46<1:17:20,  4.27s/it]
 73%|████████████████████████████████████████████████████████████████████████████████                              | 2910/3996 [3:35:50<1:16:54,  4.25s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▏                             | 2911/3996 [3:35:54<1:16:33,  4.23s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▏                             | 2912/3996 [3:35:58<1:16:18,  4.22s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▏                             | 2913/3996 [3:36:03<1:19:01,  4.38s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▏                             | 2914/3996 [3:36:07<1:18:00,  4.33s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▏                             | 2915/3996 [3:36:12<1:17:13,  4.29s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▎                             | 2916/3996 [3:36:16<1:16:36,  4.26s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▎                             | 2917/3996 [3:36:20<1:16:12,  4.24s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▎                             | 2918/3996 [3:36:24<1:15:57,  4.23s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▎                             | 2919/3996 [3:36:28<1:15:44,  4.22s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▍                             | 2920/3996 [3:36:33<1:18:33,  4.38s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▍                             | 2921/3996 [3:36:37<1:17:30,  4.33s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▍                             | 2922/3996 [3:36:41<1:16:49,  4.29s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▍                             | 2923/3996 [3:36:46<1:16:13,  4.26s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▍                             | 2924/3996 [3:36:50<1:15:50,  4.24s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▌                             | 2925/3996 [3:36:54<1:15:34,  4.23s/it]
                                                                                                                                                           
{'loss': 0.474, 'grad_norm': 0.22328485548496246, 'learning_rate': 3.509199258899603e-05, 'ppl': 1.6064, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4262.17, 'total_tokens': 55976245, 'epoch': 2.19}
+
 73%|████████████████████████████████████████████████████████████████████████████████▌                             | 2925/3996 [3:36:54<1:15:34,  4.23s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▌                             | 2926/3996 [3:36:58<1:15:18,  4.22s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▌                             | 2927/3996 [3:37:03<1:18:03,  4.38s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▌                             | 2928/3996 [3:37:07<1:17:05,  4.33s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▋                             | 2929/3996 [3:37:11<1:16:21,  4.29s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▋                             | 2930/3996 [3:37:16<1:15:45,  4.26s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▋                             | 2931/3996 [3:37:20<1:15:21,  4.25s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▋                             | 2932/3996 [3:37:24<1:15:03,  4.23s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▋                             | 2933/3996 [3:37:28<1:14:50,  4.22s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▊                             | 2934/3996 [3:37:33<1:17:32,  4.38s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▊                             | 2935/3996 [3:37:37<1:16:33,  4.33s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▊                             | 2936/3996 [3:37:41<1:15:51,  4.29s/it]
 73%|████████████████████████████████████████████████████████████████████████████████▊                             | 2937/3996 [3:37:46<1:16:08,  4.31s/it]
 74%|████████████████████████████████████████████████████████████████████████████████▉                             | 2938/3996 [3:37:50<1:15:30,  4.28s/it]
 74%|████████████████████████████████████████████████████████████████████████████████▉                             | 2939/3996 [3:37:54<1:15:04,  4.26s/it]
 74%|████████████████████████████████████████████████████████████████████████████████▉                             | 2940/3996 [3:37:58<1:14:41,  4.24s/it]
 74%|████████████████████████████████████████████████████████████████████████████████▉                             | 2941/3996 [3:38:03<1:17:20,  4.40s/it]
 74%|████████████████████████████████████████████████████████████████████████████████▉                             | 2942/3996 [3:38:07<1:16:13,  4.34s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████                             | 2943/3996 [3:38:12<1:17:42,  4.43s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████                             | 2944/3996 [3:38:16<1:16:35,  4.37s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████                             | 2945/3996 [3:38:20<1:15:43,  4.32s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████                             | 2946/3996 [3:38:25<1:15:01,  4.29s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████                             | 2947/3996 [3:38:29<1:14:30,  4.26s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▏                            | 2948/3996 [3:38:34<1:17:00,  4.41s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▏                            | 2949/3996 [3:38:38<1:15:51,  4.35s/it]
 74%|█████████████████████████████████████████████████████████████████���███████████████▏                            | 2950/3996 [3:38:42<1:14:56,  4.30s/it]
                                                                                                                                                           
{'loss': 0.4721, 'grad_norm': 0.20422938466072083, 'learning_rate': 3.3571742445268995e-05, 'ppl': 1.6034, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4339.03, 'total_tokens': 56430293, 'epoch': 2.21}
+
 74%|█████████████████████████████████████████████████████████████████████████████████▏                            | 2950/3996 [3:38:42<1:14:56,  4.30s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▏                            | 2951/3996 [3:38:46<1:14:20,  4.27s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▎                            | 2952/3996 [3:38:50<1:13:56,  4.25s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▎                            | 2953/3996 [3:38:55<1:13:37,  4.23s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▎                            | 2954/3996 [3:38:59<1:13:23,  4.23s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▎                            | 2955/3996 [3:39:04<1:15:58,  4.38s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▎                            | 2956/3996 [3:39:08<1:15:01,  4.33s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▍                            | 2957/3996 [3:39:12<1:14:16,  4.29s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▍                            | 2958/3996 [3:39:16<1:13:42,  4.26s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▍                            | 2959/3996 [3:39:20<1:13:22,  4.25s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▍                            | 2960/3996 [3:39:25<1:13:08,  4.24s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▌                            | 2961/3996 [3:39:29<1:12:56,  4.23s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▌                            | 2962/3996 [3:39:34<1:15:32,  4.38s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▌                            | 2963/3996 [3:39:38<1:14:30,  4.33s/it]
 74%|████████████████████���████████████████████████████████████████████████████████████▌                            | 2964/3996 [3:39:42<1:13:44,  4.29s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▌                            | 2965/3996 [3:39:46<1:13:11,  4.26s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▋                            | 2966/3996 [3:39:50<1:12:46,  4.24s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▋                            | 2967/3996 [3:39:55<1:12:32,  4.23s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▋                            | 2968/3996 [3:39:59<1:12:16,  4.22s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▋                            | 2969/3996 [3:40:03<1:14:53,  4.38s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▊                            | 2970/3996 [3:40:08<1:13:57,  4.33s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▊                            | 2971/3996 [3:40:12<1:13:17,  4.29s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▊                            | 2972/3996 [3:40:16<1:12:43,  4.26s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▊                            | 2973/3996 [3:40:20<1:12:21,  4.24s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▊                            | 2974/3996 [3:40:24<1:12:07,  4.23s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▉                            | 2975/3996 [3:40:29<1:11:55,  4.23s/it]
                                                                                                                                                           
{'loss': 0.4798, 'grad_norm': 0.21462033689022064, 'learning_rate': 3.2078487095649236e-05, 'ppl': 1.6158, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4274.93, 'total_tokens': 56879796, 'epoch': 2.23}
+
 74%|█████████████████████████████████████████████████████████████████████████████████▉                            | 2975/3996 [3:40:29<1:11:55,  4.23s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▉                            | 2976/3996 [3:40:33<1:14:32,  4.38s/it]
 74%|█████████████████████████████████████████████████████████████████████████████████▉                            | 2977/3996 [3:40:38<1:13:33,  4.33s/it]
 75%|█████████████████████████████████████████████████████████████████████████████████▉                            | 2978/3996 [3:40:42<1:12:50,  4.29s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████                            | 2979/3996 [3:40:46<1:12:16,  4.26s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████                            | 2980/3996 [3:40:50<1:11:55,  4.25s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████                            | 2981/3996 [3:40:54<1:11:36,  4.23s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████                            | 2982/3996 [3:40:59<1:11:18,  4.22s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████                            | 2983/3996 [3:41:03<1:13:53,  4.38s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▏                           | 2984/3996 [3:41:08<1:13:00,  4.33s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▏                           | 2985/3996 [3:41:12<1:12:18,  4.29s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▏                           | 2986/3996 [3:41:16<1:11:44,  4.26s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▏                           | 2987/3996 [3:41:20<1:11:18,  4.24s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▎                           | 2988/3996 [3:41:24<1:11:04,  4.23s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▎                           | 2989/3996 [3:41:29<1:10:52,  4.22s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▎                           | 2990/3996 [3:41:33<1:13:24,  4.38s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▎                           | 2991/3996 [3:41:38<1:12:26,  4.32s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▎                           | 2992/3996 [3:41:42<1:11:43,  4.29s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▍                           | 2993/3996 [3:41:46<1:11:10,  4.26s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▍                           | 2994/3996 [3:41:50<1:10:48,  4.24s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▍                           | 2995/3996 [3:41:54<1:10:33,  4.23s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▍                           | 2996/3996 [3:41:59<1:10:20,  4.22s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▌                           | 2997/3996 [3:42:03<1:12:56,  4.38s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▌                           | 2998/3996 [3:42:07<1:12:02,  4.33s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▌                           | 2999/3996 [3:42:12<1:11:16,  4.29s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▌                           | 3000/3996 [3:42:16<1:10:44,  4.26s/it]
                                                                                                                                                           
{'loss': 0.4733, 'grad_norm': 0.21800526976585388, 'learning_rate': 3.061283336202545e-05, 'ppl': 1.6053, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4290.7, 'total_tokens': 57329902, 'epoch': 2.25}
+
 75%|██████████████████████████████████████████████████████████████████████████████████▌                           | 3000/3996 [3:42:16<1:10:44,  4.26s/it][2025-12-29 06:32:01,639] [INFO] [axolotl.core.trainers.base.evaluate:388] [PID:3751] Running evaluation step...
+[2025-12-29 06:32:02,487] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.3769509792327881
+[2025-12-29 06:32:02,864] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.37727856636047363
+[2025-12-29 06:32:03,281] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.41676878929138184
+[2025-12-29 06:32:03,705] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:3751] generate_batches time: 0.4234030246734619
+[2025-12-29 06:32:03,706] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:3751] gather_len_batches: [72]
 
-
  0%|                                                                                                                               | 0/90 [00:00<?, ?it/s]
-
  2%|██▋                                                                                                                    | 2/90 [00:00<00:36,  2.41it/s]
-
  3%|███▉                                                                                                                   | 3/90 [00:01<00:54,  1.59it/s]
-
  4%|█████▎                                                                                                                 | 4/90 [00:02<01:00,  1.43it/s]
-
  6%|██████▌                                                                                                                | 5/90 [00:03<01:05,  1.29it/s]
-
  7%|███████▉                                                                                                               | 6/90 [00:04<01:06,  1.27it/s]
-
  8%|█████████▎                                                                                                             | 7/90 [00:05<01:08,  1.21it/s]
-
  9%|████��█████▌                                                                                                            | 8/90 [00:06<01:07,  1.22it/s]
-
 10%|███████████▉                                                                                                           | 9/90 [00:06<01:08,  1.18it/s]
-
 11%|█████████████                                                                                                         | 10/90 [00:07<01:06,  1.20it/s]
-
 12%|██████████████▍                                                                                                       | 11/90 [00:08<01:07,  1.17it/s]
-
 13%|███████████████▋                                                                                                      | 12/90 [00:09<01:05,  1.19it/s]
-
 14%|█████████████████                                                                                                     | 13/90 [00:10<01:06,  1.16it/s]
-
 16%|██████████████████▎                                                                                                   | 14/90 [00:11<01:04,  1.18it/s]
-
 17%|███████████████████▋                                                                                                  | 15/90 [00:12<01:04,  1.16it/s]
-
 18%|████████████████████▉                                                                                                 | 16/90 [00:12<01:02,  1.18it/s]
-
 19%|██████████████████████▎                                                                                               | 17/90 [00:13<01:03,  1.16it/s]
-
 20%|███████████████████████▌                                                                                              | 18/90 [00:14<01:01,  1.18it/s]
-
 21%|████████████████████████▉                                                                                             | 19/90 [00:15<01:01,  1.15it/s]
-
 22%|██████████████████████████▏                                                                                           | 20/90 [00:16<00:59,  1.17it/s]
-
 23%|███████████████████████████▌                                                                                          | 21/90 [00:17<00:59,  1.15it/s]
-
 24%|████████████████████████████▊                                                                                         | 22/90 [00:18<00:57,  1.17it/s]
-
 26%|██████████████████████████████▏                                                                                       | 23/90 [00:18<00:57,  1.16it/s]
-
 27%|███████████████████████████████▍                                                                                      | 24/90 [00:19<00:57,  1.14it/s]
-
 28%|████████████████████████████████▊                                                                                     | 25/90 [00:20<01:00,  1.08it/s]
-
 29%|██████████████████████████████████                                                                                    | 26/90 [00:21<00:55,  1.14it/s]
-
 30%|███████████████████████████████████▍                                                                                  | 27/90 [00:22<00:55,  1.14it/s]
-
 31%|████████████████████████████████████▋                                                                                 | 28/90 [00:23<00:53,  1.16it/s]
-
 32%|██████████████████████████████████████                                                                                | 29/90 [00:24<00:53,  1.13it/s]
-
 33%|███████████████████████████████████████▎                                                                              | 30/90 [00:25<00:51,  1.16it/s]
-
 34%|████████████████████████████████████████▋                                                                             | 31/90 [00:26<00:51,  1.14it/s]
-
 36%|████████��████████████████████████████████▉                                                                            | 32/90 [00:26<00:49,  1.16it/s]
-
 37%|███████████████████████████████████████████▎                                                                          | 33/90 [00:27<00:50,  1.13it/s]
-
 38%|████████████████████████████████████████████▌                                                                         | 34/90 [00:28<00:48,  1.16it/s]
-
 39%|█████████████████████████████████████████████▉                                                                        | 35/90 [00:29<00:48,  1.14it/s]
-
 40%|███████████████████████████████████████████████▏                                                                      | 36/90 [00:30<00:46,  1.16it/s]
-
 41%|████████████████████████████████████████████████▌                                                                     | 37/90 [00:31<00:46,  1.14it/s]
-
 42%|█████████████████████████████████████████████████▊                                                                    | 38/90 [00:32<00:44,  1.17it/s]
-
 43%|███████████████████████████████████████████████████▏                                                                  | 39/90 [00:32<00:44,  1.15it/s]
-
 44%|████████████████████████████████████████████████████▍                                                                 | 40/90 [00:33<00:42,  1.17it/s]
-
 46%|█████████████████████████████████████████████████████▊                                                                | 41/90 [00:34<00:42,  1.15it/s]
-
 47%|███████████████████████████████████████████████████████                                                               | 42/90 [00:35<00:41,  1.17it/s]
-
 48%|████████████████████████████████████████████████████████▍                                                             | 43/90 [00:36<00:41,  1.14it/s]
-
 49%|█████████████████████████████████████████████████████████▋                                                            | 44/90 [00:37<00:39,  1.16it/s]
-
 50%|███████████████████████████████████████████████████████████                                                           | 45/90 [00:38<00:39,  1.14it/s]
-
 51%|████████████████████████████████████████████████████████████▎                                                         | 46/90 [00:38<00:37,  1.16it/s]
-
 52%|█████████████████████████████████████████████████████████████▌                                                        | 47/90 [00:39<00:37,  1.15it/s]
-
 53%|██████████████████████████████████████████████████████████████▉                                                       | 48/90 [00:40<00:35,  1.17it/s]
-
 54%|████████████████████████████████████████████████████████████████▏                                                     | 49/90 [00:41<00:35,  1.15it/s]
-
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 50/90 [00:42<00:34,  1.17it/s]
-
 57%|█████████��████████████████████████████████████████████████████████▊                                                   | 51/90 [00:43<00:33,  1.15it/s]
-
 58%|████████████████████████████████████████████████████████████████████▏                                                 | 52/90 [00:44<00:32,  1.18it/s]
-
 59%|█████████████████████████████████████████████████████████████████████▍                                                | 53/90 [00:45<00:31,  1.16it/s]
-
 60%|██████████████████████████████████████████████████████████████████████▊                                               | 54/90 [00:45<00:30,  1.18it/s]
-
 61%|████████████████████████████████████████████████████████████████████████                                              | 55/90 [00:46<00:30,  1.15it/s]
-
 62%|█████████████████████████████████████████████████████████████████████████▍                                            | 56/90 [00:47<00:28,  1.18it/s]
-
 63%|██████████████████████████████████████████████████████████████████████████▋                                           | 57/90 [00:48<00:28,  1.16it/s]
-
 64%|████████████████████████████████████████████████████████████████████████████                                          | 58/90 [00:49<00:27,  1.18it/s]
-
 66%|█████████████████████████████████████████████████████████████████████████████▎                                        | 59/90 [00:50<00:27,  1.15it/s]
-
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 60/90 [00:51<00:25,  1.17it/s]
-
 68%|███████████████████████████████████████████████████████████████████████████████▉                                      | 61/90 [00:51<00:25,  1.15it/s]
-
 69%|█████████████████████████████████████████████████████████████████████████████████▎                                    | 62/90 [00:52<00:23,  1.17it/s]
-
 70%|██████████████████████████████████████████████████████████████████████████████████▌                                   | 63/90 [00:53<00:23,  1.15it/s]
-
 71%|███████████████████████████████████████████████████████████████████████████████████▉                                  | 64/90 [00:54<00:22,  1.17it/s]
-
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 65/90 [00:55<00:21,  1.15it/s]
-
 73%|██████████████████████████████████████████████████████████████████████████████████████▌                               | 66/90 [00:56<00:20,  1.17it/s]
-
 74%|███████████████████████████████████��███████████████████████████████████████████████████▊                              | 67/90 [00:57<00:20,  1.14it/s]
-
 76%|█████████████████████████████████████████████████████████████████████████████████████████▏                            | 68/90 [00:57<00:18,  1.16it/s]
-
 77%|██████████████████████████████████████████████████████████████████████████████████████████▍                           | 69/90 [00:58<00:18,  1.15it/s]
-
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 70/90 [00:59<00:17,  1.17it/s]
-
 79%|█████████████████████████████████████████████████████████████████████████████████████████████                         | 71/90 [01:00<00:16,  1.15it/s]
-
 80%|██████████████████████████████████████████████████████████████████████████████████████████████▍                       | 72/90 [01:01<00:15,  1.17it/s]
-
 81%|███████████████████████████████████████████████████████████████████████████████████████████████▋                      | 73/90 [01:02<00:14,  1.15it/s]
-
 82%|█████████████████████████████████████████████████████████████████████████████████████████████████                     | 74/90 [01:03<00:13,  1.18it/s]
-
 83%|██████████████████████████████████████████████████████████████████████████████████████████████████▎                   | 75/90 [01:03<00:12,  1.15it/s]
-
 84%|███████████████████████████████████████████████████████████████████████████████████████████████████▋                  | 76/90 [01:04<00:11,  1.17it/s]
-
 86%|████████████████████████████████████████████████████████████████████████████████████████████████████▉                 | 77/90 [01:05<00:11,  1.14it/s]
-
 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 78/90 [01:06<00:10,  1.17it/s]
-
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌              | 79/90 [01:07<00:09,  1.13it/s]
-
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 80/90 [01:08<00:08,  1.17it/s]
-
 90%|████████████████████████████████████████████████████████████████████████████████████████���█████████████████▏           | 81/90 [01:09<00:08,  1.11it/s]
-
 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 82/90 [01:10<00:06,  1.14it/s]
-
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 83/90 [01:10<00:06,  1.14it/s]
-
 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 84/90 [01:11<00:05,  1.16it/s]
-
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 85/90 [01:12<00:04,  1.15it/s]
-
 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 86/90 [01:13<00:03,  1.17it/s]
-
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 87/90 [01:14<00:02,  1.15it/s]
-
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 88/90 [01:15<00:01,  1.18it/s]
-
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 89/90 [01:16<00:00,  1.15it/s]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:16<00:00,  1.16it/s]
                                                                                                                                                           
-
                                                                                                                                                           
{'eval_loss': 0.5352948904037476, 'eval_runtime': 79.0505, 'eval_samples_per_second': 9.235, 'eval_steps_per_second': 2.315, 'eval_ppl': 1.708, 'memory/max_active (GiB)': 12.83, 'memory/max_allocated (GiB)': 6.85, 'memory/device_reserved (GiB)': 20.25, 'epoch': 0.96}
-
 32%|███████████████████████████████████▌                                                                           | 800/2499 [1:36:37<2:57:56,  6.28s/it]
-
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 90/90 [01:17<00:00,  1.16it/s]
-
                                                                                                                                                           [2025-12-28 12:42:16,976] [INFO] [axolotl.core.trainers.base._save:692] [PID:42410] Saving model checkpoint to ./outputs/luau-codellama-h200/checkpoint-800
+
  0%|                                                                                                                               | 0/72 [00:00<?, ?it/s]
+
  3%|███▎                                                                                                                   | 2/72 [00:01<01:08,  1.02it/s]
+
  4%|████▉                                                                                                                  | 3/72 [00:03<01:34,  1.37s/it]
+
  6%|██████▌                                                                                                                | 4/72 [00:05<01:47,  1.58s/it]
+
  7%|████████▎                                                                                                              | 5/72 [00:07<01:53,  1.70s/it]
+
  8%|█████████▉                                                                                                             | 6/72 [00:09<01:56,  1.77s/it]
+
 10%|███████████▌                                                                                                           | 7/72 [00:11<01:58,  1.82s/it]
+
 11%|█████████████▏                                                                                                         | 8/72 [00:13<01:58,  1.85s/it]
+
 12%|██████████████▉                                                                                                        | 9/72 [00:15<01:57,  1.86s/it]
+
 14%|████████████████▍                                                                                                     | 10/72 [00:17<01:56,  1.87s/it]
+
 15%|██████████████████                                                                                                    | 11/72 [00:19<01:54,  1.88s/it]
+
 17%|███████████████████▋                                                                                                  | 12/72 [00:21<01:53,  1.89s/it]
+
 18%|█████████████████████▎                                                                                                | 13/72 [00:23<01:52,  1.90s/it]
+
 19%|██████████████████████▉                                                                                               | 14/72 [00:24<01:50,  1.91s/it]
+
 21%|████████████████████████▌                                                                                             | 15/72 [00:26<01:48,  1.91s/it]
+
 22%|██████████████████████████▏                                                                                           | 16/72 [00:28<01:47,  1.92s/it]
+
 24%|███████████████████████████▊                                                                                          | 17/72 [00:30<01:45,  1.92s/it]
+
 25%|█████████████████████████████▌                                                                                        | 18/72 [00:32<01:43,  1.92s/it]
+
 26%|███████████████████████████████▏                                                                                      | 19/72 [00:35<01:50,  2.08s/it]
+
 28%|████████████████████████████████▊                                                                                     | 20/72 [00:36<01:45,  2.02s/it]
+
 29%|██████████████████████████████████▍                                                                                   | 21/72 [00:38<01:41,  1.99s/it]
+
 31%|████████████████████████████████████                                                                                  | 22/72 [00:40<01:38,  1.97s/it]
+
 32%|█████████████████████████████████████▋                                                                                | 23/72 [00:42<01:35,  1.95s/it]
+
 33%|███████████████████████████████████████▎                                                                              | 24/72 [00:44<01:33,  1.96s/it]
+
 35%|████████████████████████████████████████▉                                                                             | 25/72 [00:46<01:31,  1.95s/it]
+
 36%|██████████████████████████████████████████▌                                                                           | 26/72 [00:48<01:29,  1.94s/it]
+
 38%|████████████████████████████████████████████▎                                                                         | 27/72 [00:50<01:26,  1.93s/it]
+
 39%|█████████████████████████████████████████████▉                                                                        | 28/72 [00:52<01:24,  1.93s/it]
+
 40%|███████████████████████████████████████████████▌                                                                      | 29/72 [00:54<01:22,  1.92s/it]
+
 42%|█████████████████████████████████████████████████▏                                                                    | 30/72 [00:56<01:20,  1.91s/it]
+
 43%|██████████████████████████████████████████████████▊                                                                   | 31/72 [00:58<01:18,  1.90s/it]
+
 44%|████████████████████████████████████████████████████▍                                                                 | 32/72 [00:59<01:16,  1.91s/it]
+
 46%|██████████████████████████████████████████████████████                                                                | 33/72 [01:01<01:14,  1.91s/it]
+
 47%|███████████████████████████████████████████████████████▋                                                              | 34/72 [01:03<01:12,  1.91s/it]
+
 49%|█████████████████████████████████████████████████████████▎                                                            | 35/72 [01:05<01:10,  1.91s/it]
+
 50%|███████████████████████████████████████████████████████████                                                           | 36/72 [01:07<01:08,  1.92s/it]
+
 51%|████████████████████████████████████████████████████████████▋                                                         | 37/72 [01:09<01:07,  1.92s/it]
+
 53%|██████████████████████████████████████████████████████████████▎                                                       | 38/72 [01:11<01:05,  1.92s/it]
+
 54%|███████████████████████████████████████████████████████████████▉                                                      | 39/72 [01:13<01:03,  1.92s/it]
+
 56%|█████████████████████████████████████████████████████████████████▌                                                    | 40/72 [01:15<01:02,  1.94s/it]
+
 57%|███████████████████████████████████████████████████████████████████▏                                                  | 41/72 [01:17<00:59,  1.93s/it]
+
 58%|████████████████████████████████████████████████████████████████████▊                                                 | 42/72 [01:19<00:57,  1.92s/it]
+
 60%|██████████████████████████████████████████████████████████████████████▍                                               | 43/72 [01:21<00:55,  1.92s/it]
+
 61%|████████████████████████████████████████████████████████████���███████████                                              | 44/72 [01:22<00:53,  1.92s/it]
+
 62%|█████████████████████████████████████████████████████████████████████████▊                                            | 45/72 [01:24<00:51,  1.92s/it]
+
 64%|███████████████████████████████████████████████████████████████████████████▍                                          | 46/72 [01:26<00:49,  1.92s/it]
+
 65%|█████████████████████████████████████████████████████████████████████████████                                         | 47/72 [01:28<00:47,  1.92s/it]
+
 67%|██████████████████████████████████████████████████████████████████████████████▋                                       | 48/72 [01:30<00:46,  1.92s/it]
+
 68%|████████████████████████████████████████████████████████████████████████████████▎                                     | 49/72 [01:32<00:44,  1.92s/it]
+
 69%|█████████████████████████████████████████████████████████████████████████████████▉                                    | 50/72 [01:34<00:42,  1.92s/it]
+
 71%|███████████████████████████████████████████████████████████████████████████████████▌                                  | 51/72 [01:36<00:40,  1.91s/it]
+
 72%|█████████████████████████████████████████████████████████████████████████████████████▏                                | 52/72 [01:38<00:38,  1.91s/it]
+
 74%|██████████████████████████████████████████████████████████████████████████████████████▊                               | 53/72 [01:40<00:36,  1.91s/it]
+
 75%|████████████████████████████████████████████████████████████████████████████████████████▌                             | 54/72 [01:42<00:34,  1.91s/it]
+
 76%|██████████████████████████████████████████████████████████████████████████████████████████▏                           | 55/72 [01:44<00:32,  1.91s/it]
+
 78%|███████████████████████████████████████████████████████████████████████████████████████████▊                          | 56/72 [01:45<00:30,  1.92s/it]
+
 79%|█████████████████████████████████████████████████████████████████████████████████████████████▍                        | 57/72 [01:47<00:28,  1.92s/it]
+
 81%|███████████████████████████████████████████████████████████████████████████████████████████████                       | 58/72 [01:49<00:26,  1.92s/it]
+
 82%|████████████████████████████████████████████████████████████████████████████████████████████████▋                     | 59/72 [01:52<00:26,  2.07s/it]
+
 83%|████���█████████████████████████████████████████████████████████████████████████████████████████████▎                   | 60/72 [01:54<00:24,  2.03s/it]
+
 85%|███████████████████████████████████████████████████████████████████████████████████████████████████▉                  | 61/72 [01:56<00:21,  1.99s/it]
+
 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌                | 62/72 [01:57<00:19,  1.96s/it]
+
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎              | 63/72 [01:59<00:17,  1.94s/it]
+
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉             | 64/72 [02:01<00:15,  1.93s/it]
+
 90%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 65/72 [02:03<00:13,  1.93s/it]
+
 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 66/72 [02:05<00:11,  1.93s/it]
+
 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 67/72 [02:07<00:09,  1.92s/it]
+
 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 68/72 [02:09<00:07,  1.92s/it]
+
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████     | 69/72 [02:11<00:05,  1.92s/it]
+
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 70/72 [02:13<00:03,  1.92s/it]
+
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 71/72 [02:15<00:01,  1.92s/it]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:17<00:00,  1.93s/it]
                                                                                                                                                           
+
                                                                                                                                                           
{'eval_loss': 0.49272674322128296, 'eval_runtime': 139.4189, 'eval_samples_per_second': 5.236, 'eval_steps_per_second': 1.047, 'eval_ppl': 1.6368, 'memory/max_active (GiB)': 19.1, 'memory/max_allocated (GiB)': 19.1, 'memory/device_reserved (GiB)': 139.06, 'epoch': 2.25}
+
 75%|██████████████████████████████████████████████████████████████████████████████████▌                           | 3000/3996 [3:44:37<1:10:44,  4.26s/it]
+
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [02:17<00:00,  1.93s/it]
+
                                                                                                                                                           [2025-12-29 06:34:23,129] [INFO] [axolotl.core.trainers.base._save:692] [PID:3751] Saving model checkpoint to ./outputs/luau-codellama-h200-fast/checkpoint-3000
+
 75%|█████████████████████████████████████████████████████████████████████████████████▊                           | 3001/3996 [3:44:42<12:54:00, 46.67s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▋                           | 3002/3996 [3:44:46<9:22:05, 33.93s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▋                           | 3003/3996 [3:44:50<6:56:39, 25.18s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▋                           | 3004/3996 [3:44:55<5:12:14, 18.89s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▋                           | 3005/3996 [3:44:59<3:59:07, 14.48s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▋                           | 3006/3996 [3:45:03<3:07:57, 11.39s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▊                           | 3007/3996 [3:45:07<2:32:15,  9.24s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▊                           | 3008/3996 [3:45:11<2:07:14,  7.73s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▊                           | 3009/3996 [3:45:16<1:49:41,  6.67s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▊                           | 3010/3996 [3:45:20<1:40:02,  6.09s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▉                           | 3011/3996 [3:45:25<1:30:43,  5.53s/it]
 75%|███████████████████████████████████████████████████████��██████████████████████████▉                           | 3012/3996 [3:45:29<1:24:06,  5.13s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▉                           | 3013/3996 [3:45:33<1:19:22,  4.84s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▉                           | 3014/3996 [3:45:37<1:16:07,  4.65s/it]
 75%|██████████████████████████████████████████████████████████████████████████████████▉                           | 3015/3996 [3:45:41<1:13:52,  4.52s/it]
 75%|███████████████████████████████████████████████████████████████████████████████████                           | 3016/3996 [3:45:46<1:12:16,  4.42s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████                           | 3017/3996 [3:45:50<1:13:45,  4.52s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████                           | 3018/3996 [3:45:55<1:12:11,  4.43s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████                           | 3019/3996 [3:45:59<1:10:58,  4.36s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▏                          | 3020/3996 [3:46:03<1:10:08,  4.31s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▏                          | 3021/3996 [3:46:07<1:09:33,  4.28s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▏                          | 3022/3996 [3:46:11<1:09:04,  4.26s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▏                          | 3023/3996 [3:46:16<1:08:41,  4.24s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▏                          | 3024/3996 [3:46:20<1:11:03,  4.39s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▎                          | 3025/3996 [3:46:24<1:10:02,  4.33s/it]
                                                                                                                                                           
{'loss': 0.4705, 'grad_norm': 0.23463094234466553, 'learning_rate': 2.9175376849675073e-05, 'ppl': 1.6008, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4276.0, 'total_tokens': 59047769, 'epoch': 2.27}
+
 76%|███████████████████████████████████████████████████████████████████████████████████▎                          | 3025/3996 [3:46:24<1:10:02,  4.33s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▎                          | 3026/3996 [3:46:29<1:09:21,  4.29s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▎                          | 3027/3996 [3:46:33<1:08:45,  4.26s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▎                          | 3028/3996 [3:46:37<1:08:26,  4.24s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▍                          | 3029/3996 [3:46:41<1:08:04,  4.22s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▍                          | 3030/3996 [3:46:45<1:07:51,  4.21s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▍                          | 3031/3996 [3:46:50<1:11:40,  4.46s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▍                          | 3032/3996 [3:46:55<1:10:25,  4.38s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▍                          | 3033/3996 [3:46:59<1:09:25,  4.33s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▌                          | 3034/3996 [3:47:03<1:08:44,  4.29s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▌                          | 3035/3996 [3:47:07<1:08:17,  4.26s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▌                          | 3036/3996 [3:47:11<1:07:53,  4.24s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▌                          | 3037/3996 [3:47:16<1:07:36,  4.23s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▋                          | 3038/3996 [3:47:20<1:10:03,  4.39s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▋                          | 3039/3996 [3:47:25<1:09:09,  4.34s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▋                          | 3040/3996 [3:47:29<1:08:26,  4.30s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▋                          | 3041/3996 [3:47:33<1:07:53,  4.27s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▋                          | 3042/3996 [3:47:37<1:07:31,  4.25s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▊                          | 3043/3996 [3:47:41<1:07:13,  4.23s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▊                          | 3044/3996 [3:47:46<1:06:58,  4.22s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▊                          | 3045/3996 [3:47:50<1:09:18,  4.37s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▊                          | 3046/3996 [3:47:55<1:08:30,  4.33s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▉                          | 3047/3996 [3:47:59<1:07:46,  4.28s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▉                          | 3048/3996 [3:48:03<1:09:20,  4.39s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▉                          | 3049/3996 [3:48:08<1:09:23,  4.40s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▉                          | 3050/3996 [3:48:12<1:08:24,  4.34s/it]
                                                                                                                                                           
{'loss': 0.4761, 'grad_norm': 0.2144247442483902, 'learning_rate': 2.7766701705225194e-05, 'ppl': 1.6098, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4257.17, 'total_tokens': 59495040, 'epoch': 2.29}
+
 76%|███████████████████████████████████████████████████████████████████████████████████▉                          | 3050/3996 [3:48:12<1:08:24,  4.34s/it]
 76%|███████████████████████████████████████████████████████████████████████████████████▉                          | 3051/3996 [3:48:16<1:07:42,  4.30s/it]
 76%|████████████████████████████████████████████████████████████████████████████████████                          | 3052/3996 [3:48:21<1:09:42,  4.43s/it]
 76%|████████████████████████████████████████████████████████████████████████████████████                          | 3053/3996 [3:48:25<1:08:38,  4.37s/it]
 76%|████████████████████████████████████████████████████████████████████████████████████                          | 3054/3996 [3:48:29<1:07:44,  4.31s/it]
 76%|██████████████████���█████████████████████████████████████████████████████████████████                          | 3055/3996 [3:48:34<1:07:09,  4.28s/it]
 76%|████████████████████████████████████████████████████████████████████████████████████                          | 3056/3996 [3:48:38<1:06:42,  4.26s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▏                         | 3057/3996 [3:48:42<1:06:17,  4.24s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▏                         | 3058/3996 [3:48:46<1:05:59,  4.22s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▏                         | 3059/3996 [3:48:51<1:08:23,  4.38s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▏                         | 3060/3996 [3:48:55<1:07:34,  4.33s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▎                         | 3061/3996 [3:48:59<1:06:48,  4.29s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▎                         | 3062/3996 [3:49:04<1:06:17,  4.26s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▎                         | 3063/3996 [3:49:08<1:06:03,  4.25s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▎                         | 3064/3996 [3:49:12<1:05:46,  4.23s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▎                         | 3065/3996 [3:49:16<1:05:30,  4.22s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▍                         | 3066/3996 [3:49:21<1:09:46,  4.50s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▍                         | 3067/3996 [3:49:25<1:08:21,  4.41s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▍                         | 3068/3996 [3:49:30<1:07:18,  4.35s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▍                         | 3069/3996 [3:49:34<1:06:28,  4.30s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▌                         | 3070/3996 [3:49:38<1:05:59,  4.28s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▌                         | 3071/3996 [3:49:42<1:05:28,  4.25s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▌                         | 3072/3996 [3:49:46<1:05:10,  4.23s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▌                         | 3073/3996 [3:49:51<1:07:32,  4.39s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▌                         | 3074/3996 [3:49:55<1:06:35,  4.33s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▋                         | 3075/3996 [3:50:00<1:05:48,  4.29s/it]
                                                                                                                                                           
{'loss': 0.4576, 'grad_norm': 0.21562626957893372, 'learning_rate': 2.6387380379269623e-05, 'ppl': 1.5803, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4276.33, 'total_tokens': 59941839, 'epoch': 2.31}
+
 77%|████████████████████████████████████████████████████████████████████████████████████▋                         | 3075/3996 [3:50:00<1:05:48,  4.29s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▋                         | 3076/3996 [3:50:04<1:05:17,  4.26s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▋                         | 3077/3996 [3:50:08<1:04:55,  4.24s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▋                         | 3078/3996 [3:50:12<1:04:39,  4.23s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▊                         | 3079/3996 [3:50:16<1:04:27,  4.22s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▊                         | 3080/3996 [3:50:21<1:06:49,  4.38s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▊                         | 3081/3996 [3:50:25<1:05:59,  4.33s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▊                         | 3082/3996 [3:50:30<1:05:19,  4.29s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▊                         | 3083/3996 [3:50:34<1:04:48,  4.26s/it]
 77%|█████████████████████���██████████████████████████████████████████████████████████████▉                         | 3084/3996 [3:50:38<1:04:32,  4.25s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▉                         | 3085/3996 [3:50:42<1:04:15,  4.23s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▉                         | 3086/3996 [3:50:46<1:04:02,  4.22s/it]
 77%|████████████████████████████████████████████████████████████████████████████████████▉                         | 3087/3996 [3:50:51<1:06:20,  4.38s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████                         | 3088/3996 [3:50:55<1:05:31,  4.33s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████                         | 3089/3996 [3:51:00<1:04:48,  4.29s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████                         | 3090/3996 [3:51:04<1:04:15,  4.26s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████                         | 3091/3996 [3:51:08<1:03:53,  4.24s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████                         | 3092/3996 [3:51:12<1:03:40,  4.23s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████▏                        | 3093/3996 [3:51:16<1:03:30,  4.22s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████▏                        | 3094/3996 [3:51:21<1:05:43,  4.37s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████▏                        | 3095/3996 [3:51:25<1:04:54,  4.32s/it]
 77%|█████████████████████████████████████████████████████████████████████████████████████▏                        | 3096/3996 [3:51:29<1:04:17,  4.29s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▎                        | 3097/3996 [3:51:34<1:03:49,  4.26s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▎                        | 3098/3996 [3:51:38<1:03:29,  4.24s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▎                        | 3099/3996 [3:51:42<1:03:13,  4.23s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▎                        | 3100/3996 [3:51:46<1:03:03,  4.22s/it]
                                                                                                                                                           
{'loss': 0.4578, 'grad_norm': 0.2173856496810913, 'learning_rate': 2.5037973393739433e-05, 'ppl': 1.5806, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4284.31, 'total_tokens': 60392267, 'epoch': 2.33}
+
 78%|█████████████████████████████████████████████████████████████████████████████████████▎                        | 3100/3996 [3:51:46<1:03:03,  4.22s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▎                        | 3101/3996 [3:51:51<1:05:21,  4.38s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▍                        | 3102/3996 [3:51:55<1:04:29,  4.33s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▍                        | 3103/3996 [3:51:59<1:03:49,  4.29s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▍                        | 3104/3996 [3:52:04<1:03:18,  4.26s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▍                        | 3105/3996 [3:52:08<1:03:01,  4.24s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▌                        | 3106/3996 [3:52:12<1:02:43,  4.23s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▌                        | 3107/3996 [3:52:16<1:02:33,  4.22s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▌                        | 3108/3996 [3:52:21<1:04:46,  4.38s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▌                        | 3109/3996 [3:52:25<1:03:51,  4.32s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▌                        | 3110/3996 [3:52:29<1:03:15,  4.28s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▋                        | 3111/3996 [3:52:34<1:02:51,  4.26s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▋                        | 3112/3996 [3:52:38<1:02:33,  4.25s/it]
 78%|█████████���███████████████████████████████████████████████████████████████████████████▋                        | 3113/3996 [3:52:42<1:02:15,  4.23s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▋                        | 3114/3996 [3:52:46<1:02:02,  4.22s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▋                        | 3115/3996 [3:52:51<1:04:19,  4.38s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▊                        | 3116/3996 [3:52:55<1:03:30,  4.33s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▊                        | 3117/3996 [3:52:59<1:02:50,  4.29s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▊                        | 3118/3996 [3:53:03<1:02:19,  4.26s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▊                        | 3119/3996 [3:53:08<1:02:03,  4.25s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▉                        | 3120/3996 [3:53:12<1:01:47,  4.23s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▉                        | 3121/3996 [3:53:16<1:03:14,  4.34s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▉                        | 3122/3996 [3:53:21<1:04:58,  4.46s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▉                        | 3123/3996 [3:53:25<1:03:44,  4.38s/it]
 78%|█████████████████████████████████████████████████████████████████████████████████████▉                        | 3124/3996 [3:53:30<1:02:49,  4.32s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████                        | 3125/3996 [3:53:34<1:02:07,  4.28s/it]
                                                                                                                                                           
{'loss': 0.4652, 'grad_norm': 0.21864096820354462, 'learning_rate': 2.3719029114120716e-05, 'ppl': 1.5923, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4250.05, 'total_tokens': 60836393, 'epoch': 2.34}
+
 78%|██████████████████████████████████████████████████████████████████████████████████████                        | 3125/3996 [3:53:34<1:02:07,  4.28s/it]
 78%|█████████████████████████████████��████████████████████████████████████████████████████                        | 3126/3996 [3:53:38<1:01:43,  4.26s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████                        | 3127/3996 [3:53:42<1:01:23,  4.24s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████                        | 3128/3996 [3:53:46<1:01:07,  4.23s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▏                       | 3129/3996 [3:53:51<1:03:17,  4.38s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▏                       | 3130/3996 [3:53:55<1:02:29,  4.33s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▏                       | 3131/3996 [3:54:00<1:01:49,  4.29s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▏                       | 3132/3996 [3:54:04<1:01:15,  4.25s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▏                       | 3133/3996 [3:54:08<1:01:01,  4.24s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▎                       | 3134/3996 [3:54:12<1:00:45,  4.23s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▎                       | 3135/3996 [3:54:16<1:00:31,  4.22s/it]
 78%|██████████████████████████████████████████████████████████████████████████████████████▎                       | 3136/3996 [3:54:21<1:02:41,  4.37s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▎                       | 3137/3996 [3:54:25<1:01:56,  4.33s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▍                       | 3138/3996 [3:54:29<1:01:24,  4.29s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▍                       | 3139/3996 [3:54:34<1:00:55,  4.27s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▍                       | 3140/3996 [3:54:38<1:00:37,  4.25s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▍                       | 3141/3996 [3:54:42<1:00:18,  4.23s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▍                       | 3142/3996 [3:54:46<1:00:07,  4.22s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▌                       | 3143/3996 [3:54:51<1:02:15,  4.38s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▌                       | 3144/3996 [3:54:55<1:01:30,  4.33s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▌                       | 3145/3996 [3:54:59<1:00:51,  4.29s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▌                       | 3146/3996 [3:55:04<1:00:20,  4.26s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▋                       | 3147/3996 [3:55:08<1:00:00,  4.24s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▏                       | 3148/3996 [3:55:12<59:44,  4.23s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▎                       | 3149/3996 [3:55:16<59:34,  4.22s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▋                       | 3150/3996 [3:55:21<1:01:39,  4.37s/it]
                                                                                                                                                           
{'loss': 0.4721, 'grad_norm': 0.22768662869930267, 'learning_rate': 2.2431083526612373e-05, 'ppl': 1.6034, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3777.63, 'total_tokens': 61282878, 'epoch': 2.36}
+
 79%|██████████████████████████████████████████████████████████████████████████████████████▋                       | 3150/3996 [3:55:21<1:01:39,  4.37s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▋                       | 3151/3996 [3:55:25<1:00:54,  4.33s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▊                       | 3152/3996 [3:55:29<1:00:15,  4.28s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▎                       | 3153/3996 [3:55:34<59:48,  4.26s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▍                       | 3154/3996 [3:55:38<59:33,  4.24s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▍                       | 3155/3996 [3:55:42<59:16,  4.23s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▍                       | 3156/3996 [3:55:46<59:04,  4.22s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▉                       | 3157/3996 [3:55:51<1:01:12,  4.38s/it]
 79%|██████████████████████████████████████████████████████████████████████████████████████▉                       | 3158/3996 [3:55:55<1:00:23,  4.32s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▌                       | 3159/3996 [3:55:59<59:46,  4.29s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▌                       | 3160/3996 [3:56:03<59:19,  4.26s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▌                       | 3161/3996 [3:56:08<59:03,  4.24s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▌                       | 3162/3996 [3:56:12<58:48,  4.23s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▋                       | 3163/3996 [3:56:16<58:37,  4.22s/it]
 79%|███████████████████████████████████████████████████████████████████████████████████████                       | 3164/3996 [3:56:21<1:00:39,  4.37s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▋                       | 3165/3996 [3:56:25<59:52,  4.32s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▋                       | 3166/3996 [3:56:29<59:16,  4.29s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▊                       | 3167/3996 [3:56:33<58:51,  4.26s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▊                       | 3168/3996 [3:56:38<58:36,  4.25s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▊                       | 3169/3996 [3:56:42<58:18,  4.23s/it]
 79%|████████████████████████████████��███████████████████████████████████████████████████████▊                       | 3170/3996 [3:56:46<58:08,  4.22s/it]
 79%|███████████████████████████████████████████████████████████████████████████████████████▎                      | 3171/3996 [3:56:51<1:00:12,  4.38s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▉                       | 3172/3996 [3:56:55<59:26,  4.33s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▉                       | 3173/3996 [3:56:59<58:50,  4.29s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▉                       | 3174/3996 [3:57:03<58:22,  4.26s/it]
 79%|████████████████████████████████████████████████████████████████████████████████████████▉                       | 3175/3996 [3:57:08<58:00,  4.24s/it]
                                                                                                                                                           
{'loss': 0.486, 'grad_norm': 0.20161285996437073, 'learning_rate': 2.1174660020314696e-05, 'ppl': 1.6258, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4279.7, 'total_tokens': 61731262, 'epoch': 2.38}
+
 79%|████████████████████████████████████████████████████████████████████████████████████████▉                       | 3175/3996 [3:57:08<58:00,  4.24s/it]
 79%|█████████████████████████████████████████████████████████████████████████████████████████                       | 3176/3996 [3:57:12<57:47,  4.23s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████                       | 3177/3996 [3:57:16<57:38,  4.22s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████                       | 3178/3996 [3:57:21<59:38,  4.37s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████                       | 3179/3996 [3:57:25<58:54,  4.33s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▏                      | 3180/3996 [3:57:29<58:15,  4.28s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▏                      | 3181/3996 [3:57:33<57:46,  4.25s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▏                      | 3182/3996 [3:57:37<57:28,  4.24s/it]
 80%|████████████████████████████���████████████████████████████████████████████████████████████▏                      | 3183/3996 [3:57:42<57:14,  4.22s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▏                      | 3184/3996 [3:57:46<57:00,  4.21s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▎                      | 3185/3996 [3:57:51<59:06,  4.37s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▎                      | 3186/3996 [3:57:55<58:45,  4.35s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▎                      | 3187/3996 [3:57:59<58:04,  4.31s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▎                      | 3188/3996 [3:58:03<57:33,  4.27s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▍                      | 3189/3996 [3:58:08<57:14,  4.26s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▍                      | 3190/3996 [3:58:12<56:57,  4.24s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▍                      | 3191/3996 [3:58:16<56:45,  4.23s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▍                      | 3192/3996 [3:58:21<58:42,  4.38s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▍                      | 3193/3996 [3:58:25<57:57,  4.33s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▌                      | 3194/3996 [3:58:29<57:21,  4.29s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▌                      | 3195/3996 [3:58:33<56:55,  4.26s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▌                      | 3196/3996 [3:58:37<56:37,  4.25s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▌                      | 3197/3996 [3:58:42<56:46,  4.26s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▋                      | 3198/3996 [3:58:46<56:25,  4.24s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▋                      | 3199/3996 [3:58:51<58:22,  4.40s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▋                      | 3200/3996 [3:58:55<57:36,  4.34s/it]
                                                                                                                                                           
{'loss': 0.4822, 'grad_norm': 0.2132490575313568, 'learning_rate': 1.9950269174537007e-05, 'ppl': 1.6196, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4245.52, 'total_tokens': 62179118, 'epoch': 2.4}
+
 80%|█████████████████████████████████████████████████████████████████████████████████████████▋                      | 3200/3996 [3:58:55<57:36,  4.34s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▋                      | 3201/3996 [3:58:59<56:59,  4.30s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▋                      | 3202/3996 [3:59:03<56:30,  4.27s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▊                      | 3203/3996 [3:59:08<56:10,  4.25s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▊                      | 3204/3996 [3:59:12<55:55,  4.24s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▊                      | 3205/3996 [3:59:16<55:44,  4.23s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▊                      | 3206/3996 [3:59:21<57:40,  4.38s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▉                      | 3207/3996 [3:59:25<56:59,  4.33s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▉                      | 3208/3996 [3:59:29<56:35,  4.31s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▉                      | 3209/3996 [3:59:33<56:07,  4.28s/it]
 80%|█████████████████████████████████████████████████████████████████████████████████████████▉                      | 3210/3996 [3:59:38<57:04,  4.36s/it]
 80%|██████████████████████████████████████████��██████████████████████████████████████████████▉                      | 3211/3996 [3:59:43<58:04,  4.44s/it]
 80%|██████████████████████████████████████████████████████████████████████████████████████████                      | 3212/3996 [3:59:47<57:03,  4.37s/it]
 80%|██████████████████████████████████████████████████████████████████████████████████████████                      | 3213/3996 [3:59:52<58:32,  4.49s/it]
 80%|██████████████████████████████████████████████████████████████████████████████████████████                      | 3214/3996 [3:59:56<57:24,  4.40s/it]
 80%|██████████████████████████████████████████████████████████████████████████████████████████                      | 3215/3996 [4:00:00<56:30,  4.34s/it]
 80%|██████████████████████████████████████████████████████████████████████████████████████████▏                     | 3216/3996 [4:00:04<55:50,  4.30s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▏                     | 3217/3996 [4:00:08<55:26,  4.27s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▏                     | 3218/3996 [4:00:13<55:02,  4.25s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▏                     | 3219/3996 [4:00:17<54:47,  4.23s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▎                     | 3220/3996 [4:00:21<56:41,  4.38s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▎                     | 3221/3996 [4:00:26<55:58,  4.33s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▎                     | 3222/3996 [4:00:30<55:25,  4.30s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▎                     | 3223/3996 [4:00:34<54:57,  4.27s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▎                     | 3224/3996 [4:00:38<54:38,  4.25s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▍                     | 3225/3996 [4:00:42<54:23,  4.23s/it]
                                                                                                                                                           
{'loss': 0.49, 'grad_norm': 0.24689531326293945, 'learning_rate': 1.8758408551311047e-05, 'ppl': 1.6323, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4237.8, 'total_tokens': 62624159, 'epoch': 2.42}
+
 81%|██████████████████████████████████████████████████████████████████████████████████████████▍                     | 3225/3996 [4:00:42<54:23,  4.23s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▍                     | 3226/3996 [4:00:47<54:12,  4.22s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▍                     | 3227/3996 [4:00:51<56:07,  4.38s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▍                     | 3228/3996 [4:00:56<55:23,  4.33s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▌                     | 3229/3996 [4:01:00<56:01,  4.38s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▌                     | 3230/3996 [4:01:04<55:12,  4.32s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▌                     | 3231/3996 [4:01:09<54:38,  4.29s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▌                     | 3232/3996 [4:01:13<54:15,  4.26s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▌                     | 3233/3996 [4:01:17<53:54,  4.24s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▋                     | 3234/3996 [4:01:22<56:03,  4.41s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▋                     | 3235/3996 [4:01:26<55:14,  4.36s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▋                     | 3236/3996 [4:01:30<55:17,  4.37s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▋                     | 3237/3996 [4:01:35<54:51,  4.34s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▊                     | 3238/3996 [4:01:39<54:16,  4.30s/it]
 81%|█████████████████████████████████████████���████████████████████████████████████████████████▊                     | 3239/3996 [4:01:43<54:05,  4.29s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▊                     | 3240/3996 [4:01:47<53:48,  4.27s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▊                     | 3241/3996 [4:01:52<55:34,  4.42s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▊                     | 3242/3996 [4:01:56<54:46,  4.36s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▉                     | 3243/3996 [4:02:01<54:06,  4.31s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▉                     | 3244/3996 [4:02:05<53:36,  4.28s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▉                     | 3245/3996 [4:02:09<53:17,  4.26s/it]
 81%|██████████████████████████████████████████████████████████████████████████████████████████▉                     | 3246/3996 [4:02:13<53:01,  4.24s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████                     | 3247/3996 [4:02:17<52:47,  4.23s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████                     | 3248/3996 [4:02:22<54:35,  4.38s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████                     | 3249/3996 [4:02:26<53:54,  4.33s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████                     | 3250/3996 [4:02:30<53:23,  4.29s/it]
                                                                                                                                                           
{'loss': 0.4654, 'grad_norm': 0.2103738784790039, 'learning_rate': 1.7599562493193867e-05, 'ppl': 1.5927, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4234.16, 'total_tokens': 63069936, 'epoch': 2.44}
+
 81%|███████████████████████████████████████████████████████████████████████████████████████████                     | 3250/3996 [4:02:30<53:23,  4.29s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████                     | 3251/3996 [4:02:35<52:58,  4.27s/it]
 81%|███████████████████���███████████████████████████████████████████████████████████████████████▏                    | 3252/3996 [4:02:39<52:39,  4.25s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████▏                    | 3253/3996 [4:02:43<52:26,  4.23s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████▏                    | 3254/3996 [4:02:47<52:12,  4.22s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████▏                    | 3255/3996 [4:02:52<54:02,  4.38s/it]
 81%|███████████████████████████████████████████████████████████████████████████████████████████▎                    | 3256/3996 [4:02:56<53:20,  4.33s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▎                    | 3257/3996 [4:03:00<52:46,  4.28s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▎                    | 3258/3996 [4:03:05<52:19,  4.25s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▎                    | 3259/3996 [4:03:09<52:02,  4.24s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▎                    | 3260/3996 [4:03:13<51:49,  4.23s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▍                    | 3261/3996 [4:03:18<53:18,  4.35s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▍                    | 3262/3996 [4:03:22<54:41,  4.47s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▍                    | 3263/3996 [4:03:27<55:45,  4.56s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▍                    | 3264/3996 [4:03:31<54:22,  4.46s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▌                    | 3265/3996 [4:03:36<53:16,  4.37s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▌                    | 3266/3996 [4:03:40<52:37,  4.33s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▌                    | 3267/3996 [4:03:44<52:03,  4.29s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▌                    | 3268/3996 [4:03:48<51:42,  4.26s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▌                    | 3269/3996 [4:03:53<53:20,  4.40s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▋                    | 3270/3996 [4:03:57<52:33,  4.34s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▋                    | 3271/3996 [4:04:01<51:56,  4.30s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▋                    | 3272/3996 [4:04:05<51:27,  4.26s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▋                    | 3273/3996 [4:04:10<51:07,  4.24s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▊                    | 3274/3996 [4:04:14<50:53,  4.23s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▊                    | 3275/3996 [4:04:18<50:46,  4.23s/it]
                                                                                                                                                           
{'loss': 0.4575, 'grad_norm': 0.2107544094324112, 'learning_rate': 1.6474201926443267e-05, 'ppl': 1.5801, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4198.5, 'total_tokens': 63512282, 'epoch': 2.46}
+
 82%|███████████████████████████████████████████████████████████████████████████████████████████▊                    | 3275/3996 [4:04:18<50:46,  4.23s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▊                    | 3276/3996 [4:04:23<52:31,  4.38s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▊                    | 3277/3996 [4:04:27<51:51,  4.33s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▉                    | 3278/3996 [4:04:31<51:20,  4.29s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▉                    | 3279/3996 [4:04:35<50:52,  4.26s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▉                    | 3280/3996 [4:04:40<50:36,  4.24s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▉                    | 3281/3996 [4:04:44<50:44,  4.26s/it]
 82%|███████████████████████████████████████████████████████████████████████████████████████████▉                    | 3282/3996 [4:04:48<50:27,  4.24s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████                    | 3283/3996 [4:04:53<52:11,  4.39s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████                    | 3284/3996 [4:04:57<51:27,  4.34s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████                    | 3285/3996 [4:05:01<50:52,  4.29s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████                    | 3286/3996 [4:05:05<50:27,  4.26s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▏                   | 3287/3996 [4:05:10<50:11,  4.25s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▏                   | 3288/3996 [4:05:14<49:58,  4.24s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▏                   | 3289/3996 [4:05:18<49:43,  4.22s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▏                   | 3290/3996 [4:05:23<51:55,  4.41s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▏                   | 3291/3996 [4:05:27<51:22,  4.37s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▎                   | 3292/3996 [4:05:31<50:51,  4.33s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▎                   | 3293/3996 [4:05:36<50:27,  4.31s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▎                   | 3294/3996 [4:05:40<50:04,  4.28s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▎                   | 3295/3996 [4:05:44<49:54,  4.27s/it]
 82%|████████████████████████████████████████████████████████████████████████████████████████████▍                   | 3296/3996 [4:05:48<49:34,  4.25s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▍                   | 3297/3996 [4:05:53<51:11,  4.39s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▍                   | 3298/3996 [4:05:57<50:28,  4.34s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▍                   | 3299/3996 [4:06:01<49:53,  4.30s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▍                   | 3300/3996 [4:06:06<49:27,  4.26s/it]
                                                                                                                                                           
{'loss': 0.4654, 'grad_norm': 0.211527019739151, 'learning_rate': 1.5382784169644925e-05, 'ppl': 1.5927, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4266.77, 'total_tokens': 63959153, 'epoch': 2.48}
+
 83%|████████████████████████████████████████████████████████████████████████████████████████████▍                   | 3300/3996 [4:06:06<49:27,  4.26s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▌                   | 3301/3996 [4:06:10<49:39,  4.29s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▌                   | 3302/3996 [4:06:14<49:19,  4.26s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▌                   | 3303/3996 [4:06:18<48:59,  4.24s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▌                   | 3304/3996 [4:06:23<50:35,  4.39s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▋                   | 3305/3996 [4:06:27<49:51,  4.33s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▋                   | 3306/3996 [4:06:32<49:18,  4.29s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▋                   | 3307/3996 [4:06:36<48:52,  4.26s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▋                   | 3308/3996 [4:06:40<48:38,  4.24s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▋                   | 3309/3996 [4:06:44<48:24,  4.23s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▊                   | 3310/3996 [4:06:48<48:11,  4.21s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▊                   | 3311/3996 [4:06:53<49:53,  4.37s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▊                   | 3312/3996 [4:06:57<49:13,  4.32s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▊                   | 3313/3996 [4:07:01<48:42,  4.28s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▉                   | 3314/3996 [4:07:06<48:18,  4.25s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▉                   | 3315/3996 [4:07:10<48:03,  4.23s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▉                   | 3316/3996 [4:07:14<47:50,  4.22s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▉                   | 3317/3996 [4:07:18<47:39,  4.21s/it]
 83%|████████████████████████████████████████████████████████████████████████████████████████████▉                   | 3318/3996 [4:07:23<49:21,  4.37s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████                   | 3319/3996 [4:07:27<48:43,  4.32s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████                   | 3320/3996 [4:07:31<48:12,  4.28s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████                   | 3321/3996 [4:07:35<47:50,  4.25s/it]
 83%|██████████████████████████████████████████████████████████████████��██████████████████████████                   | 3322/3996 [4:07:40<47:34,  4.24s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▏                  | 3323/3996 [4:07:44<47:20,  4.22s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▏                  | 3324/3996 [4:07:48<48:29,  4.33s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▏                  | 3325/3996 [4:07:53<49:42,  4.44s/it]
                                                                                                                                                           
{'loss': 0.4601, 'grad_norm': 0.22054381668567657, 'learning_rate': 1.4325752747869626e-05, 'ppl': 1.5842, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3810.76, 'total_tokens': 64408084, 'epoch': 2.49}
+
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▏                  | 3325/3996 [4:07:53<49:42,  4.44s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▏                  | 3326/3996 [4:07:57<48:48,  4.37s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▏                  | 3327/3996 [4:08:02<48:03,  4.31s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▎                  | 3328/3996 [4:08:06<47:34,  4.27s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▎                  | 3329/3996 [4:08:10<47:13,  4.25s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▎                  | 3330/3996 [4:08:14<47:01,  4.24s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▎                  | 3331/3996 [4:08:18<46:48,  4.22s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▍                  | 3332/3996 [4:08:23<48:22,  4.37s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▍                  | 3333/3996 [4:08:27<47:44,  4.32s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▍                  | 3334/3996 [4:08:31<47:14,  4.28s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▍                  | 3335/3996 [4:08:36<46:47,  4.25s/it]
 83%|█████████████████████████████████████████████████████████████████████████████████████████████▌                  | 3336/3996 [4:08:40<46:27,  4.22s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌                  | 3337/3996 [4:08:44<47:41,  4.34s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌                  | 3338/3996 [4:08:49<47:07,  4.30s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌                  | 3339/3996 [4:08:53<48:26,  4.42s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▌                  | 3340/3996 [4:08:57<47:35,  4.35s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▋                  | 3341/3996 [4:09:02<46:58,  4.30s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▋                  | 3342/3996 [4:09:06<46:29,  4.26s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▋                  | 3343/3996 [4:09:10<46:08,  4.24s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▋                  | 3344/3996 [4:09:14<45:53,  4.22s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▊                  | 3345/3996 [4:09:18<45:41,  4.21s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▊                  | 3346/3996 [4:09:23<47:17,  4.37s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▊                  | 3347/3996 [4:09:27<46:40,  4.31s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▊                  | 3348/3996 [4:09:31<46:10,  4.28s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▊                  | 3349/3996 [4:09:36<45:47,  4.25s/it]
 84%|███��█████████████████████████████████████████████████████████████████████████████████████████▉                  | 3350/3996 [4:09:40<45:47,  4.25s/it]
                                                                                                                                                           
{'loss': 0.4594, 'grad_norm': 0.21859121322631836, 'learning_rate': 1.3303537212435469e-05, 'ppl': 1.5831, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4141.07, 'total_tokens': 64850022, 'epoch': 2.51}
+
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▉                  | 3350/3996 [4:09:40<45:47,  4.25s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▉                  | 3351/3996 [4:09:44<45:33,  4.24s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▉                  | 3352/3996 [4:09:48<45:21,  4.23s/it]
 84%|█████████████████████████████████████████████████████████████████████████████████████████████▉                  | 3353/3996 [4:09:53<47:14,  4.41s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████                  | 3354/3996 [4:09:57<46:34,  4.35s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████                  | 3355/3996 [4:10:02<47:39,  4.46s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████                  | 3356/3996 [4:10:06<46:44,  4.38s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████                  | 3357/3996 [4:10:11<46:22,  4.35s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████                  | 3358/3996 [4:10:15<45:49,  4.31s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▏                 | 3359/3996 [4:10:19<45:22,  4.27s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▏                 | 3360/3996 [4:10:24<46:47,  4.41s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▏                 | 3361/3996 [4:10:28<46:02,  4.35s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▏                 | 3362/3996 [4:10:32<45:27,  4.30s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▎                 | 3363/3996 [4:10:36<44:59,  4.26s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▎                 | 3364/3996 [4:10:41<44:47,  4.25s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▎                 | 3365/3996 [4:10:45<44:31,  4.23s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▎                 | 3366/3996 [4:10:49<44:18,  4.22s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▎                 | 3367/3996 [4:10:54<45:54,  4.38s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▍                 | 3368/3996 [4:10:58<45:16,  4.33s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▍                 | 3369/3996 [4:11:02<45:52,  4.39s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▍                 | 3370/3996 [4:11:07<45:08,  4.33s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▍                 | 3371/3996 [4:11:11<44:48,  4.30s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▌                 | 3372/3996 [4:11:15<44:24,  4.27s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▌                 | 3373/3996 [4:11:19<44:13,  4.26s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▌                 | 3374/3996 [4:11:24<45:38,  4.40s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▌                 | 3375/3996 [4:11:28<44:56,  4.34s/it]
                                                                                                                                                           
{'loss': 0.4635, 'grad_norm': 0.22012574970722198, 'learning_rate': 1.231655296634906e-05, 'ppl': 1.5896, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4210.83, 'total_tokens': 65292271, 'epoch': 2.53}
+
 84%|██████��███████████████████████████████████████████████████████████████████████████████████████▌                 | 3375/3996 [4:11:28<44:56,  4.34s/it]
 84%|██████████████████████████████████████████████████████████████████████████████████████████████▌                 | 3376/3996 [4:11:32<44:23,  4.30s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋                 | 3377/3996 [4:11:37<43:59,  4.26s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋                 | 3378/3996 [4:11:41<43:43,  4.25s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋                 | 3379/3996 [4:11:45<43:29,  4.23s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▋                 | 3380/3996 [4:11:49<43:19,  4.22s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊                 | 3381/3996 [4:11:54<44:48,  4.37s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊                 | 3382/3996 [4:11:58<44:11,  4.32s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊                 | 3383/3996 [4:12:02<43:41,  4.28s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊                 | 3384/3996 [4:12:06<43:22,  4.25s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▊                 | 3385/3996 [4:12:11<43:12,  4.24s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▉                 | 3386/3996 [4:12:15<43:02,  4.23s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▉                 | 3387/3996 [4:12:19<42:50,  4.22s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▉                 | 3388/3996 [4:12:24<44:20,  4.38s/it]
 85%|██████████████████████████████████████████████████████████████████████████████████████████████▉                 | 3389/3996 [4:12:28<43:47,  4.33s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████                 | 3390/3996 [4:12:32<43:17,  4.29s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████                 | 3391/3996 [4:12:37<43:58,  4.36s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████                 | 3392/3996 [4:12:41<43:24,  4.31s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████                 | 3393/3996 [4:12:45<42:59,  4.28s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▏                | 3394/3996 [4:12:49<42:39,  4.25s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▏                | 3395/3996 [4:12:54<44:00,  4.39s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▏                | 3396/3996 [4:12:58<43:22,  4.34s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▏                | 3397/3996 [4:13:02<42:52,  4.29s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▏                | 3398/3996 [4:13:07<42:26,  4.26s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▎                | 3399/3996 [4:13:11<42:10,  4.24s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▎                | 3400/3996 [4:13:15<41:56,  4.22s/it]
                                                                                                                                                           
{'loss': 0.4809, 'grad_norm': 0.21981129050254822, 'learning_rate': 1.1365201095496048e-05, 'ppl': 1.6175, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4233.11, 'total_tokens': 65735025, 'epoch': 2.55}
+
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▎                | 3400/3996 [4:13:15<41:56,  4.22s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▎                | 3401/3996 [4:13:19<41:48,  4.22s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▎                | 3402/3996 [4:13:24<43:15,  4.37s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▍                | 3403/3996 [4:13:28<42:38,  4.32s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▍                | 3404/3996 [4:13:32<42:13,  4.28s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▍                | 3405/3996 [4:13:37<41:51,  4.25s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▍                | 3406/3996 [4:13:41<41:39,  4.24s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▍                | 3407/3996 [4:13:45<41:28,  4.22s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▌                | 3408/3996 [4:13:49<41:18,  4.21s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▌                | 3409/3996 [4:13:54<42:43,  4.37s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▌                | 3410/3996 [4:13:58<42:08,  4.32s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▌                | 3411/3996 [4:14:02<41:40,  4.27s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▋                | 3412/3996 [4:14:06<41:20,  4.25s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▋                | 3413/3996 [4:14:11<41:06,  4.23s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▋                | 3414/3996 [4:14:15<40:53,  4.22s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▋                | 3415/3996 [4:14:19<40:46,  4.21s/it]
 85%|███████████████████████████████████████████████████████████████████████████████████████████████▋                | 3416/3996 [4:14:24<42:12,  4.37s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊                | 3417/3996 [4:14:28<41:35,  4.31s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊                | 3418/3996 [4:14:32<41:09,  4.27s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊                | 3419/3996 [4:14:36<40:50,  4.25s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▊                | 3420/3996 [4:14:40<40:38,  4.23s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉                | 3421/3996 [4:14:45<40:25,  4.22s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉                | 3422/3996 [4:14:49<40:18,  4.21s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉                | 3423/3996 [4:14:54<41:43,  4.37s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉                | 3424/3996 [4:14:58<41:10,  4.32s/it]
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉                | 3425/3996 [4:15:02<40:45,  4.28s/it]
                                                                                                                                                           
{'loss': 0.4605, 'grad_norm': 0.22363677620887756, 'learning_rate': 1.0449868205649649e-05, 'ppl': 1.5849, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4245.88, 'total_tokens': 66180426, 'epoch': 2.57}
+
 86%|███████████████████████████████████████████████████████████████████████████████████████████████▉                | 3425/3996 [4:15:02<40:45,  4.28s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████                | 3426/3996 [4:15:06<40:25,  4.25s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████                | 3427/3996 [4:15:10<40:12,  4.24s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████                | 3428/3996 [4:15:15<40:00,  4.23s/it]
 86%|██████████████████████████████████████████████��█████████████████████████████████████████████████                | 3429/3996 [4:15:19<39:51,  4.22s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▏               | 3430/3996 [4:15:23<41:14,  4.37s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▏               | 3431/3996 [4:15:28<40:40,  4.32s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▏               | 3432/3996 [4:15:32<40:15,  4.28s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▏               | 3433/3996 [4:15:36<39:55,  4.25s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▏               | 3434/3996 [4:15:40<39:43,  4.24s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▎               | 3435/3996 [4:15:44<39:31,  4.23s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▎               | 3436/3996 [4:15:49<39:22,  4.22s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▎               | 3437/3996 [4:15:53<40:46,  4.38s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▎               | 3438/3996 [4:15:58<40:12,  4.32s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▍               | 3439/3996 [4:16:02<39:45,  4.28s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▍               | 3440/3996 [4:16:06<39:25,  4.25s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▍               | 3441/3996 [4:16:10<39:11,  4.24s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▍               | 3442/3996 [4:16:14<39:00,  4.22s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▌               | 3443/3996 [4:16:19<38:49,  4.21s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▌               | 3444/3996 [4:16:23<40:14,  4.37s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▌               | 3445/3996 [4:16:27<39:38,  4.32s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▌               | 3446/3996 [4:16:32<39:15,  4.28s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▌               | 3447/3996 [4:16:36<38:55,  4.25s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▋               | 3448/3996 [4:16:40<38:44,  4.24s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▋               | 3449/3996 [4:16:44<38:33,  4.23s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▋               | 3450/3996 [4:16:48<38:24,  4.22s/it]
                                                                                                                                                           
{'loss': 0.4661, 'grad_norm': 0.21145139634609222, 'learning_rate': 9.570926265363789e-06, 'ppl': 1.5938, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4280.85, 'total_tokens': 66629602, 'epoch': 2.59}
+
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▋               | 3450/3996 [4:16:48<38:24,  4.22s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▋               | 3451/3996 [4:16:53<39:42,  4.37s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▊               | 3452/3996 [4:16:57<39:10,  4.32s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▊               | 3453/3996 [4:17:02<38:42,  4.28s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▊               | 3454/3996 [4:17:06<38:23,  4.25s/it]
 86%|████████████████████████████████████████████████████████████████████████████████████████████████▊               | 3455/3996 [4:17:10<38:11,  4.24s/it]
 86%|██████████████████████████████████████████��█████████████████████████████████████████████████████▊               | 3456/3996 [4:17:14<37:59,  4.22s/it]
 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉               | 3457/3996 [4:17:18<37:50,  4.21s/it]
 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉               | 3458/3996 [4:17:23<39:09,  4.37s/it]
 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉               | 3459/3996 [4:17:27<38:38,  4.32s/it]
 87%|████████████████████████████████████████████████████████████████████████████████████████████████▉               | 3460/3996 [4:17:31<38:13,  4.28s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████               | 3461/3996 [4:17:36<39:00,  4.38s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████               | 3462/3996 [4:17:40<38:33,  4.33s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████               | 3463/3996 [4:17:45<38:08,  4.29s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████               | 3464/3996 [4:17:49<37:47,  4.26s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████               | 3465/3996 [4:17:53<38:59,  4.41s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▏              | 3466/3996 [4:17:58<38:21,  4.34s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▏              | 3467/3996 [4:18:02<37:54,  4.30s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▏              | 3468/3996 [4:18:06<37:31,  4.26s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▏              | 3469/3996 [4:18:10<37:15,  4.24s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▎              | 3470/3996 [4:18:14<37:03,  4.23s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▎              | 3471/3996 [4:18:19<37:02,  4.23s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▎              | 3472/3996 [4:18:23<38:16,  4.38s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▎              | 3473/3996 [4:18:28<37:40,  4.32s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▎              | 3474/3996 [4:18:32<37:16,  4.29s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▍              | 3475/3996 [4:18:36<36:57,  4.26s/it]
                                                                                                                                                           
{'loss': 0.46, 'grad_norm': 0.2377360314130783, 'learning_rate': 8.728732454814203e-06, 'ppl': 1.5841, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4258.29, 'total_tokens': 67075180, 'epoch': 2.61}
+
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▍              | 3475/3996 [4:18:36<36:57,  4.26s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▍              | 3476/3996 [4:18:40<36:44,  4.24s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▍              | 3477/3996 [4:18:44<36:32,  4.22s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▍              | 3478/3996 [4:18:49<36:25,  4.22s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▌              | 3479/3996 [4:18:53<37:40,  4.37s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▌              | 3480/3996 [4:18:57<37:09,  4.32s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▌              | 3481/3996 [4:19:02<36:45,  4.28s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▌              | 3482/3996 [4:19:06<36:27,  4.25s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▌              | 3483/3996 [4:19:10<36:14,  4.24s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▋              | 3484/3996 [4:19:14<36:03,  4.23s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▋              | 3485/3996 [4:19:19<36:53,  4.33s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▋              | 3486/3996 [4:19:24<37:49,  4.45s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▋              | 3487/3996 [4:19:28<37:05,  4.37s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▊              | 3488/3996 [4:19:32<36:31,  4.31s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▊              | 3489/3996 [4:19:36<36:06,  4.27s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▊              | 3490/3996 [4:19:40<35:51,  4.25s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▊              | 3491/3996 [4:19:44<35:36,  4.23s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▊              | 3492/3996 [4:19:49<35:24,  4.21s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▉              | 3493/3996 [4:19:53<36:35,  4.36s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▉              | 3494/3996 [4:19:58<36:05,  4.31s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▉              | 3495/3996 [4:20:02<35:40,  4.27s/it]
 87%|█████████████████████████████████████████████████████████████████████████████████████████████████▉              | 3496/3996 [4:20:06<35:22,  4.24s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████              | 3497/3996 [4:20:10<35:08,  4.23s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████              | 3498/3996 [4:20:14<34:57,  4.21s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████              | 3499/3996 [4:20:18<34:48,  4.20s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████              | 3500/3996 [4:20:23<36:07,  4.37s/it]
                                                                                                                                                           
{'loss': 0.4734, 'grad_norm': 0.22640903294086456, 'learning_rate': 7.923629020649448e-06, 'ppl': 1.6054, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3732.79, 'total_tokens': 67519655, 'epoch': 2.63}
+
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████              | 3500/3996 [4:20:23<36:07,  4.37s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏             | 3501/3996 [4:20:27<35:35,  4.31s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏             | 3502/3996 [4:20:32<35:09,  4.27s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏             | 3503/3996 [4:20:36<34:50,  4.24s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏             | 3504/3996 [4:20:40<34:38,  4.22s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▏             | 3505/3996 [4:20:44<34:26,  4.21s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▎             | 3506/3996 [4:20:48<34:20,  4.20s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▎             | 3507/3996 [4:20:53<35:30,  4.36s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▎             | 3508/3996 [4:20:57<35:01,  4.31s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▎             | 3509/3996 [4:21:01<34:39,  4.27s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▍             | 3510/3996 [4:21:06<34:21,  4.24s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▍             | 3511/3996 [4:21:10<34:10,  4.23s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▍             | 3512/3996 [4:21:14<33:57,  4.21s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▍             | 3513/3996 [4:21:18<33:51,  4.21s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▍             | 3514/3996 [4:21:24<36:38,  4.56s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▌             | 3515/3996 [4:21:28<35:37,  4.44s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▌             | 3516/3996 [4:21:32<34:55,  4.37s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▌             | 3517/3996 [4:21:36<34:27,  4.32s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▌             | 3518/3996 [4:21:40<34:08,  4.29s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▋             | 3519/3996 [4:21:44<33:49,  4.25s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▋             | 3520/3996 [4:21:49<33:37,  4.24s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▋             | 3521/3996 [4:21:53<34:42,  4.38s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▋             | 3522/3996 [4:21:58<34:12,  4.33s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▋             | 3523/3996 [4:22:02<33:46,  4.29s/it]
 88%|████████████████████████████████████████████████████���█████████████████████████████████████████████▊             | 3524/3996 [4:22:06<33:27,  4.25s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▊             | 3525/3996 [4:22:10<33:14,  4.24s/it]
                                                                                                                                                           
{'loss': 0.4701, 'grad_norm': 0.2617396414279938, 'learning_rate': 7.155943136910193e-06, 'ppl': 1.6002, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4208.25, 'total_tokens': 67960790, 'epoch': 2.64}
+
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▊             | 3525/3996 [4:22:10<33:14,  4.24s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▊             | 3526/3996 [4:22:14<33:04,  4.22s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▊             | 3527/3996 [4:22:19<32:56,  4.21s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▉             | 3528/3996 [4:22:23<34:03,  4.37s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▉             | 3529/3996 [4:22:27<33:34,  4.31s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▉             | 3530/3996 [4:22:32<33:10,  4.27s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▉             | 3531/3996 [4:22:36<32:54,  4.25s/it]
 88%|██████████████████████████████████████████████████████████████████████████████████████████████████▉             | 3532/3996 [4:22:40<32:43,  4.23s/it]
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████             | 3533/3996 [4:22:44<32:33,  4.22s/it]
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████             | 3534/3996 [4:22:48<32:24,  4.21s/it]
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████             | 3535/3996 [4:22:54<34:33,  4.50s/it]
 88%|███████████████████████████████████████████████████████████████████████████████████████████████████             | 3536/3996 [4:22:58<33:47,  4.41s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏            | 3537/3996 [4:23:02<33:11,  4.34s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏            | 3538/3996 [4:23:06<32:46,  4.29s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏            | 3539/3996 [4:23:10<32:28,  4.26s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏            | 3540/3996 [4:23:15<32:14,  4.24s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏            | 3541/3996 [4:23:19<32:05,  4.23s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎            | 3542/3996 [4:23:23<33:09,  4.38s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎            | 3543/3996 [4:23:28<32:38,  4.32s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎            | 3544/3996 [4:23:32<32:16,  4.28s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▎            | 3545/3996 [4:23:36<31:59,  4.26s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▍            | 3546/3996 [4:23:40<31:46,  4.24s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▍            | 3547/3996 [4:23:44<31:35,  4.22s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▍            | 3548/3996 [4:23:49<31:28,  4.22s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▍            | 3549/3996 [4:23:53<32:32,  4.37s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▍            | 3550/3996 [4:23:58<32:06,  4.32s/it]
                                                                                                                                                           
{'loss': 0.4754, 'grad_norm': 0.2119966447353363, 'learning_rate': 6.425986772073922e-06, 'ppl': 1.6087, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4238.47, 'total_tokens': 68405914, 'epoch': 2.66}
+
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▍            | 3550/3996 [4:23:58<32:06,  4.32s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▌            | 3551/3996 [4:24:02<31:45,  4.28s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▌            | 3552/3996 [4:24:06<31:29,  4.25s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▌            | 3553/3996 [4:24:10<31:17,  4.24s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▌            | 3554/3996 [4:24:14<31:07,  4.22s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▋            | 3555/3996 [4:24:19<30:59,  4.22s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▋            | 3556/3996 [4:24:23<32:03,  4.37s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▋            | 3557/3996 [4:24:27<31:35,  4.32s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▋            | 3558/3996 [4:24:32<31:13,  4.28s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▊            | 3559/3996 [4:24:36<30:56,  4.25s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▊            | 3560/3996 [4:24:40<30:43,  4.23s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▊            | 3561/3996 [4:24:44<30:32,  4.21s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▊            | 3562/3996 [4:24:48<30:26,  4.21s/it]
 89%|██████████████���████████████████████████████████████████████████████████████████████████████████████▊            | 3563/3996 [4:24:53<31:32,  4.37s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▉            | 3564/3996 [4:24:57<31:06,  4.32s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▉            | 3565/3996 [4:25:01<30:45,  4.28s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▉            | 3566/3996 [4:25:06<30:27,  4.25s/it]
 89%|███████████████████████████████████████████████████████████████████████████████████████████████████▉            | 3567/3996 [4:25:10<30:17,  4.24s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████            | 3568/3996 [4:25:14<30:06,  4.22s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████            | 3569/3996 [4:25:18<29:58,  4.21s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████            | 3570/3996 [4:25:23<31:00,  4.37s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████            | 3571/3996 [4:25:27<30:33,  4.31s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████            | 3572/3996 [4:25:31<30:09,  4.27s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 3573/3996 [4:25:35<29:52,  4.24s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 3574/3996 [4:25:40<29:43,  4.23s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 3575/3996 [4:25:44<29:32,  4.21s/it]
                                                                                                                                                           
{'loss': 0.4536, 'grad_norm': 0.21404898166656494, 'learning_rate': 5.734056562278634e-06, 'ppl': 1.574, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4293.79, 'total_tokens': 68854437, 'epoch': 2.68}
+
 89%|█████████████████████████���██████████████████████████████████████████████████████████████████████████▏           | 3575/3996 [4:25:44<29:32,  4.21s/it]
 89%|████████████████████████████████████████████████████████████████████████████████████████████████████▏           | 3576/3996 [4:25:48<29:27,  4.21s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎           | 3577/3996 [4:25:53<30:29,  4.37s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎           | 3578/3996 [4:25:57<30:03,  4.31s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎           | 3579/3996 [4:26:01<29:42,  4.28s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎           | 3580/3996 [4:26:05<29:26,  4.25s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▎           | 3581/3996 [4:26:10<29:14,  4.23s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▍           | 3582/3996 [4:26:14<29:02,  4.21s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▍           | 3583/3996 [4:26:18<28:54,  4.20s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▍           | 3584/3996 [4:26:23<29:55,  4.36s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▍           | 3585/3996 [4:26:27<29:30,  4.31s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 3586/3996 [4:26:31<29:06,  4.26s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 3587/3996 [4:26:35<28:49,  4.23s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 3588/3996 [4:26:39<28:41,  4.22s/it]
 90%|█████████████████████████████████████████████████████████████���██████████████████████████████████████▌           | 3589/3996 [4:26:43<28:32,  4.21s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 3590/3996 [4:26:48<28:26,  4.20s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▋           | 3591/3996 [4:26:52<29:23,  4.36s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▋           | 3592/3996 [4:26:57<28:58,  4.30s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▋           | 3593/3996 [4:27:01<28:39,  4.27s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▋           | 3594/3996 [4:27:05<28:23,  4.24s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▊           | 3595/3996 [4:27:09<28:12,  4.22s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▊           | 3596/3996 [4:27:13<28:04,  4.21s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▊           | 3597/3996 [4:27:17<27:56,  4.20s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▊           | 3598/3996 [4:27:22<28:55,  4.36s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▊           | 3599/3996 [4:27:27<28:51,  4.36s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▉           | 3600/3996 [4:27:31<28:36,  4.33s/it]
                                                                                                                                                           
{'loss': 0.4726, 'grad_norm': 0.207435742020607, 'learning_rate': 5.080433690777353e-06, 'ppl': 1.6042, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4137.98, 'total_tokens': 69296241, 'epoch': 2.7}
+
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▉           | 3600/3996 [4:27:31<28:36,  4.33s/it]
 90%|██████████████████████████████████████████████████████████████████���█████████████████████████████████▉           | 3601/3996 [4:27:35<28:12,  4.28s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▉           | 3602/3996 [4:27:39<27:58,  4.26s/it]
 90%|████████████████████████████████████████████████████████████████████████████████████████████████████▉           | 3603/3996 [4:27:43<27:45,  4.24s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████           | 3604/3996 [4:27:48<27:35,  4.22s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████           | 3605/3996 [4:27:52<28:28,  4.37s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████           | 3606/3996 [4:27:56<28:03,  4.32s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████           | 3607/3996 [4:28:01<27:43,  4.28s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏          | 3608/3996 [4:28:05<28:19,  4.38s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏          | 3609/3996 [4:28:09<27:53,  4.32s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏          | 3610/3996 [4:28:14<27:33,  4.28s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏          | 3611/3996 [4:28:18<27:17,  4.25s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏          | 3612/3996 [4:28:23<28:14,  4.41s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎          | 3613/3996 [4:28:27<27:44,  4.35s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎          | 3614/3996 [4:28:31<27:20,  4.30s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████��███▎          | 3615/3996 [4:28:35<27:03,  4.26s/it]
 90%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎          | 3616/3996 [4:28:39<26:50,  4.24s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍          | 3617/3996 [4:28:44<26:40,  4.22s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍          | 3618/3996 [4:28:48<26:32,  4.21s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍          | 3619/3996 [4:28:52<27:24,  4.36s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍          | 3620/3996 [4:28:57<27:00,  4.31s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍          | 3621/3996 [4:29:01<26:41,  4.27s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 3622/3996 [4:29:05<26:27,  4.24s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 3623/3996 [4:29:09<26:17,  4.23s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 3624/3996 [4:29:13<26:10,  4.22s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 3625/3996 [4:29:18<26:03,  4.21s/it]
                                                                                                                                                           
{'loss': 0.4626, 'grad_norm': 0.23212255537509918, 'learning_rate': 4.465383773672127e-06, 'ppl': 1.5882, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4195.93, 'total_tokens': 69736200, 'epoch': 2.72}
+
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌          | 3625/3996 [4:29:18<26:03,  4.21s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋          | 3626/3996 [4:29:22<26:56,  4.37s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋          | 3627/3996 [4:29:27<26:32,  4.32s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋          | 3628/3996 [4:29:31<26:13,  4.27s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋          | 3629/3996 [4:29:35<25:58,  4.25s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋          | 3630/3996 [4:29:39<25:49,  4.23s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊          | 3631/3996 [4:29:43<25:41,  4.22s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊          | 3632/3996 [4:29:48<25:35,  4.22s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊          | 3633/3996 [4:29:52<26:27,  4.37s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊          | 3634/3996 [4:29:56<26:03,  4.32s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉          | 3635/3996 [4:30:01<25:45,  4.28s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉          | 3636/3996 [4:30:05<25:31,  4.25s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉          | 3637/3996 [4:30:09<25:21,  4.24s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉          | 3638/3996 [4:30:13<25:10,  4.22s/it]
 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉          | 3639/3996 [4:30:17<25:04,  4.21s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████          | 3640/3996 [4:30:22<25:55,  4.37s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████          | 3641/3996 [4:30:26<25:32,  4.32s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████          | 3642/3996 [4:30:31<25:14,  4.28s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████          | 3643/3996 [4:30:35<25:00,  4.25s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 3644/3996 [4:30:39<24:51,  4.24s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 3645/3996 [4:30:43<24:42,  4.22s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 3646/3996 [4:30:47<24:36,  4.22s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 3647/3996 [4:30:52<25:25,  4.37s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▏         | 3648/3996 [4:30:56<25:02,  4.32s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎         | 3649/3996 [4:31:00<24:45,  4.28s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎         | 3650/3996 [4:31:05<24:30,  4.25s/it]
                                                                                                                                                           
{'loss': 0.4652, 'grad_norm': 0.24078768491744995, 'learning_rate': 3.889156751974343e-06, 'ppl': 1.5923, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4260.72, 'total_tokens': 70181446, 'epoch': 2.74}
+
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎         | 3650/3996 [4:31:05<24:30,  4.25s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎         | 3651/3996 [4:31:09<24:21,  4.24s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎         | 3652/3996 [4:31:13<24:12,  4.22s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍         | 3653/3996 [4:31:17<24:05,  4.21s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍         | 3654/3996 [4:31:22<24:53,  4.37s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍         | 3655/3996 [4:31:26<24:31,  4.31s/it]
 91%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍         | 3656/3996 [4:31:30<24:13,  4.28s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍         | 3657/3996 [4:31:34<24:00,  4.25s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌         | 3658/3996 [4:31:39<23:50,  4.23s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌         | 3659/3996 [4:31:43<23:41,  4.22s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌         | 3660/3996 [4:31:47<23:33,  4.21s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌         | 3661/3996 [4:31:52<24:18,  4.36s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋         | 3662/3996 [4:31:56<24:22,  4.38s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋         | 3663/3996 [4:32:00<23:58,  4.32s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋         | 3664/3996 [4:32:05<23:40,  4.28s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▋         | 3665/3996 [4:32:09<23:27,  4.25s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 3666/3996 [4:32:13<23:16,  4.23s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 3667/3996 [4:32:17<23:07,  4.22s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 3668/3996 [4:32:22<23:53,  4.37s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 3669/3996 [4:32:26<23:31,  4.32s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊         | 3670/3996 [4:32:30<23:13,  4.27s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉         | 3671/3996 [4:32:34<22:58,  4.24s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉         | 3672/3996 [4:32:39<22:48,  4.22s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉         | 3673/3996 [4:32:43<22:40,  4.21s/it]
 92%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉         | 3674/3996 [4:32:47<22:32,  4.20s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████         | 3675/3996 [4:32:52<23:18,  4.36s/it]
                                                                                                                                                           
{'loss': 0.4603, 'grad_norm': 0.20761160552501678, 'learning_rate': 3.3519867900349113e-06, 'ppl': 1.5845, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3783.05, 'total_tokens': 70627535, 'epoch': 2.76}
+
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████         | 3675/3996 [4:32:52<23:18,  4.36s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████         | 3676/3996 [4:32:56<22:58,  4.31s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████         | 3677/3996 [4:33:00<22:42,  4.27s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████         | 3678/3996 [4:33:04<22:29,  4.24s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████         | 3679/3996 [4:33:08<22:20,  4.23s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏        | 3680/3996 [4:33:13<22:11,  4.21s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏        | 3681/3996 [4:33:17<22:04,  4.21s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏        | 3682/3996 [4:33:21<22:50,  4.36s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▏        | 3683/3996 [4:33:26<22:29,  4.31s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 3684/3996 [4:33:30<22:12,  4.27s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 3685/3996 [4:33:34<21:58,  4.24s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 3686/3996 [4:33:38<21:48,  4.22s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 3687/3996 [4:33:42<21:40,  4.21s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 3688/3996 [4:33:47<21:35,  4.21s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍        | 3689/3996 [4:33:51<22:17,  4.36s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍        | 3690/3996 [4:33:55<21:57,  4.30s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍        | 3691/3996 [4:34:00<21:41,  4.27s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍        | 3692/3996 [4:34:04<21:28,  4.24s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌        | 3693/3996 [4:34:08<21:20,  4.23s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌        | 3694/3996 [4:34:12<21:12,  4.21s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌        | 3695/3996 [4:34:16<21:05,  4.21s/it]
 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌        | 3696/3996 [4:34:21<21:48,  4.36s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌        | 3697/3996 [4:34:25<21:30,  4.31s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋        | 3698/3996 [4:34:29<21:14,  4.28s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋        | 3699/3996 [4:34:34<21:01,  4.25s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋        | 3700/3996 [4:34:38<20:53,  4.23s/it]
                                                                                                                                                           
{'loss': 0.459, 'grad_norm': 0.2079222947359085, 'learning_rate': 2.8540921803855926e-06, 'ppl': 1.5825, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4196.92, 'total_tokens': 71068359, 'epoch': 2.78}
+
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋        | 3700/3996 [4:34:38<20:53,  4.23s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋        | 3701/3996 [4:34:42<20:45,  4.22s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 3702/3996 [4:34:46<20:39,  4.21s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 3703/3996 [4:34:51<21:20,  4.37s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 3704/3996 [4:34:55<21:00,  4.32s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 3705/3996 [4:34:59<20:44,  4.28s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊        | 3706/3996 [4:35:04<20:35,  4.26s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉        | 3707/3996 [4:35:08<20:25,  4.24s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉        | 3708/3996 [4:35:12<20:17,  4.23s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉        | 3709/3996 [4:35:16<20:09,  4.21s/it]
 93%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉        | 3710/3996 [4:35:21<20:49,  4.37s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████        | 3711/3996 [4:35:25<20:31,  4.32s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████        | 3712/3996 [4:35:29<20:15,  4.28s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████        | 3713/3996 [4:35:33<20:01,  4.25s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████        | 3714/3996 [4:35:38<19:52,  4.23s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████        | 3715/3996 [4:35:42<19:44,  4.21s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 3716/3996 [4:35:46<19:37,  4.21s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 3717/3996 [4:35:51<20:36,  4.43s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 3718/3996 [4:35:55<20:11,  4.36s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 3719/3996 [4:35:59<19:54,  4.31s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎       | 3720/3996 [4:36:04<19:39,  4.27s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎       | 3721/3996 [4:36:08<19:28,  4.25s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎       | 3722/3996 [4:36:12<19:19,  4.23s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎       | 3723/3996 [4:36:16<19:11,  4.22s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍       | 3724/3996 [4:36:21<19:49,  4.37s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍       | 3725/3996 [4:36:25<19:29,  4.32s/it]
                                                                                                                                                           
{'loss': 0.4692, 'grad_norm': 0.23349842429161072, 'learning_rate': 2.395675255030383e-06, 'ppl': 1.5987, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4216.77, 'total_tokens': 71509553, 'epoch': 2.79}
+
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍       | 3725/3996 [4:36:25<19:29,  4.32s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍       | 3726/3996 [4:36:29<19:15,  4.28s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍       | 3727/3996 [4:36:33<19:03,  4.25s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍       | 3728/3996 [4:36:38<18:55,  4.24s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌       | 3729/3996 [4:36:42<18:46,  4.22s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌       | 3730/3996 [4:36:46<18:40,  4.21s/it]
 93%|██████████████████████████████████████████████████████████████████████████���█████████████████████████████▌       | 3731/3996 [4:36:51<19:16,  4.37s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▌       | 3732/3996 [4:36:55<18:58,  4.31s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋       | 3733/3996 [4:36:59<18:45,  4.28s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋       | 3734/3996 [4:37:03<18:33,  4.25s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋       | 3735/3996 [4:37:08<18:37,  4.28s/it]
 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋       | 3736/3996 [4:37:12<18:25,  4.25s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋       | 3737/3996 [4:37:16<18:18,  4.24s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊       | 3738/3996 [4:37:21<19:15,  4.48s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊       | 3739/3996 [4:37:25<18:49,  4.39s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊       | 3740/3996 [4:37:29<18:29,  4.33s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊       | 3741/3996 [4:37:34<18:13,  4.29s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉       | 3742/3996 [4:37:38<18:02,  4.26s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉       | 3743/3996 [4:37:42<17:52,  4.24s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉       | 3744/3996 [4:37:46<17:43,  4.22s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉       | 3745/3996 [4:37:51<18:17,  4.37s/it]
 94%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉       | 3746/3996 [4:37:55<17:58,  4.32s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████       | 3747/3996 [4:37:59<17:45,  4.28s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████       | 3748/3996 [4:38:03<17:33,  4.25s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████       | 3749/3996 [4:38:08<17:24,  4.23s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████       | 3750/3996 [4:38:12<17:16,  4.21s/it]
                                                                                                                                                           
{'loss': 0.4626, 'grad_norm': 0.2154284566640854, 'learning_rate': 1.9769223032228724e-06, 'ppl': 1.5882, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4278.81, 'total_tokens': 71956413, 'epoch': 2.81}
+
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████       | 3750/3996 [4:38:12<17:16,  4.21s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏      | 3751/3996 [4:38:16<17:08,  4.20s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏      | 3752/3996 [4:38:21<17:42,  4.35s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏      | 3753/3996 [4:38:25<17:26,  4.31s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏      | 3754/3996 [4:38:29<17:12,  4.26s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏      | 3755/3996 [4:38:33<17:00,  4.24s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎      | 3756/3996 [4:38:37<16:51,  4.22s/it]
 94%|██████████████████████████████████████���██████████████████████████████████████████████████████████████████▎      | 3757/3996 [4:38:42<16:44,  4.20s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎      | 3758/3996 [4:38:46<16:38,  4.20s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎      | 3759/3996 [4:38:50<17:11,  4.35s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 3760/3996 [4:38:55<16:55,  4.30s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 3761/3996 [4:38:59<16:41,  4.26s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 3762/3996 [4:39:03<16:31,  4.24s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 3763/3996 [4:39:07<16:24,  4.22s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 3764/3996 [4:39:11<16:16,  4.21s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌      | 3765/3996 [4:39:16<16:11,  4.21s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌      | 3766/3996 [4:39:20<16:43,  4.37s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌      | 3767/3996 [4:39:25<16:28,  4.32s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▌      | 3768/3996 [4:39:29<16:16,  4.28s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋      | 3769/3996 [4:39:33<16:05,  4.25s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋      | 3770/3996 [4:39:37<15:56,  4.23s/it]
 94%|███████████████████████████��█████████████████████████████████████████████████████████████████████████████▋      | 3771/3996 [4:39:42<16:06,  4.29s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋      | 3772/3996 [4:39:46<15:55,  4.26s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋      | 3773/3996 [4:39:50<16:21,  4.40s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊      | 3774/3996 [4:39:55<16:02,  4.34s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊      | 3775/3996 [4:39:59<15:48,  4.29s/it]
                                                                                                                                                           
{'loss': 0.4757, 'grad_norm': 0.2559005916118622, 'learning_rate': 1.5980034957628231e-06, 'ppl': 1.6091, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4161.74, 'total_tokens': 72391979, 'epoch': 2.83}
+
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊      | 3775/3996 [4:39:59<15:48,  4.29s/it]
 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊      | 3776/3996 [4:40:03<15:37,  4.26s/it]
 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊      | 3777/3996 [4:40:07<15:28,  4.24s/it]
 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉      | 3778/3996 [4:40:11<15:20,  4.22s/it]
 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉      | 3779/3996 [4:40:16<15:14,  4.21s/it]
 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉      | 3780/3996 [4:40:20<15:44,  4.37s/it]
 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉      | 3781/3996 [4:40:24<15:28,  4.32s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████      | 3782/3996 [4:40:29<15:16,  4.28s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████      | 3783/3996 [4:40:33<15:05,  4.25s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████      | 3784/3996 [4:40:37<14:57,  4.23s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████      | 3785/3996 [4:40:41<14:51,  4.22s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████      | 3786/3996 [4:40:45<14:44,  4.21s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏     | 3787/3996 [4:40:50<15:13,  4.37s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏     | 3788/3996 [4:40:54<14:57,  4.31s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏     | 3789/3996 [4:40:59<15:05,  4.37s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏     | 3790/3996 [4:41:03<14:49,  4.32s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎     | 3791/3996 [4:41:07<14:37,  4.28s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎     | 3792/3996 [4:41:11<14:27,  4.25s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎     | 3793/3996 [4:41:16<14:18,  4.23s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎     | 3794/3996 [4:41:20<14:45,  4.38s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎     | 3795/3996 [4:41:25<14:30,  4.33s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████���███████████▍     | 3796/3996 [4:41:29<14:17,  4.29s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍     | 3797/3996 [4:41:33<14:07,  4.26s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍     | 3798/3996 [4:41:37<13:59,  4.24s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍     | 3799/3996 [4:41:41<13:51,  4.22s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌     | 3800/3996 [4:41:46<13:45,  4.21s/it]
                                                                                                                                                           
{'loss': 0.4823, 'grad_norm': 0.21530191600322723, 'learning_rate': 1.2590728158430431e-06, 'ppl': 1.6198, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4264.41, 'total_tokens': 72837687, 'epoch': 2.85}
+
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌     | 3800/3996 [4:41:46<13:45,  4.21s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌     | 3801/3996 [4:41:50<14:11,  4.37s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌     | 3802/3996 [4:41:54<13:57,  4.32s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌     | 3803/3996 [4:41:59<13:45,  4.28s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌     | 3804/3996 [4:42:03<13:35,  4.25s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3805/3996 [4:42:07<13:28,  4.23s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3806/3996 [4:42:11<13:21,  4.22s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3807/3996 [4:42:15<13:15,  4.21s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3808/3996 [4:42:20<13:41,  4.37s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 3809/3996 [4:42:24<13:27,  4.32s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 3810/3996 [4:42:29<13:16,  4.28s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 3811/3996 [4:42:33<13:06,  4.25s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 3812/3996 [4:42:37<12:58,  4.23s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 3813/3996 [4:42:41<12:52,  4.22s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3814/3996 [4:42:45<12:46,  4.21s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3815/3996 [4:42:50<13:10,  4.37s/it]
 95%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3816/3996 [4:42:54<12:56,  4.31s/it]
 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3817/3996 [4:42:58<12:44,  4.27s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████     | 3818/3996 [4:43:03<12:35,  4.25s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████     | 3819/3996 [4:43:07<12:28,  4.23s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████     | 3820/3996 [4:43:11<12:21,  4.22s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████     | 3821/3996 [4:43:15<12:15,  4.21s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████     | 3822/3996 [4:43:20<12:38,  4.36s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3823/3996 [4:43:24<12:25,  4.31s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3824/3996 [4:43:28<12:13,  4.27s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3825/3996 [4:43:32<12:05,  4.24s/it]
                                                                                                                                                           
{'loss': 0.468, 'grad_norm': 0.22042331099510193, 'learning_rate': 9.602679964744288e-07, 'ppl': 1.5968, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4254.73, 'total_tokens': 73282095, 'epoch': 2.87}
+
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3825/3996 [4:43:32<12:05,  4.24s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3826/3996 [4:43:37<11:57,  4.22s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3827/3996 [4:43:41<11:51,  4.21s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3828/3996 [4:43:45<11:45,  4.20s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3829/3996 [4:43:50<12:06,  4.35s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3830/3996 [4:43:54<11:53,  4.30s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍    | 3831/3996 [4:43:58<11:43,  4.26s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍    | 3832/3996 [4:44:02<11:35,  4.24s/it]
 96%|██████████████████████████████████████████████████████████████████████████████████████████████���████████████▍    | 3833/3996 [4:44:06<11:28,  4.22s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍    | 3834/3996 [4:44:11<11:22,  4.21s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍    | 3835/3996 [4:44:15<11:16,  4.20s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3836/3996 [4:44:19<11:37,  4.36s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3837/3996 [4:44:24<11:25,  4.31s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3838/3996 [4:44:28<11:14,  4.27s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3839/3996 [4:44:32<11:06,  4.24s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋    | 3840/3996 [4:44:36<10:59,  4.23s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋    | 3841/3996 [4:44:40<10:53,  4.22s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3842/3996 [4:44:45<10:48,  4.21s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3843/3996 [4:44:50<11:29,  4.51s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3844/3996 [4:44:54<11:11,  4.41s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋     | 3845/3996 [4:44:58<10:56,  4.35s/it]
 96%|███████████████████████████████████████████████████████████████████████████████████��█████████████████████████████████████████████████▊     | 3846/3996 [4:45:02<10:45,  4.30s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 3847/3996 [4:45:07<10:36,  4.27s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 3848/3996 [4:45:11<10:28,  4.25s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3849/3996 [4:45:15<10:21,  4.23s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3850/3996 [4:45:20<10:39,  4.38s/it]
                                                                                                                                                                                      
{'loss': 0.4651, 'grad_norm': 0.22293563187122345, 'learning_rate': 7.017104645146599e-07, 'ppl': 1.5922, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 3747.29, 'total_tokens': 73724795, 'epoch': 2.89}
+
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3850/3996 [4:45:20<10:39,  4.38s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3851/3996 [4:45:24<10:27,  4.33s/it]
 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉     | 3852/3996 [4:45:28<10:16,  4.28s/it]
 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████     | 3853/3996 [4:45:32<10:08,  4.26s/it]
 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████     | 3854/3996 [4:45:36<10:01,  4.24s/it]
 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████��███████     | 3855/3996 [4:45:41<09:55,  4.22s/it]
 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3856/3996 [4:45:45<09:49,  4.21s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3857/3996 [4:45:50<10:07,  4.37s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3858/3996 [4:45:54<09:56,  4.32s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏    | 3859/3996 [4:45:58<09:46,  4.28s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3860/3996 [4:46:02<09:37,  4.25s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3861/3996 [4:46:06<09:31,  4.24s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3862/3996 [4:46:11<09:25,  4.22s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 3863/3996 [4:46:15<09:20,  4.21s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍    | 3864/3996 [4:46:19<09:36,  4.37s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍    | 3865/3996 [4:46:24<09:25,  4.32s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍    | 3866/3996 [4:46:28<09:16,  4.28s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3867/3996 [4:46:32<09:07,  4.25s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3868/3996 [4:46:36<09:01,  4.23s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3869/3996 [4:46:40<08:55,  4.22s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌    | 3870/3996 [4:46:45<08:50,  4.21s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋    | 3871/3996 [4:46:49<09:07,  4.38s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋    | 3872/3996 [4:46:54<08:56,  4.33s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋    | 3873/3996 [4:46:58<08:46,  4.28s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊    | 3874/3996 [4:47:02<08:38,  4.25s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊    | 3875/3996 [4:47:06<08:32,  4.24s/it]
                                                                                                                                                                                      
{'loss': 0.4598, 'grad_norm': 0.20620891451835632, 'learning_rate': 4.83505291323405e-07, 'ppl': 1.5838, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4202.85, 'total_tokens': 74165185, 'epoch': 2.91}
+
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊    | 3875/3996 [4:47:06<08:32,  4.24s/it]
 97%|██████████████████████████████████████████████████████████��███████████████████████████████████████████████████████████████████████████▊    | 3876/3996 [4:47:11<08:34,  4.29s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊    | 3877/3996 [4:47:15<08:27,  4.26s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉    | 3878/3996 [4:47:19<08:39,  4.40s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉    | 3879/3996 [4:47:24<08:28,  4.34s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉    | 3880/3996 [4:47:28<08:18,  4.30s/it]
 97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉    | 3881/3996 [4:47:32<08:10,  4.26s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 3882/3996 [4:47:36<08:03,  4.24s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 3883/3996 [4:47:40<07:57,  4.22s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████    | 3884/3996 [4:47:45<07:52,  4.22s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏   | 3885/3996 [4:47:49<08:04,  4.37s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏   | 3886/3996 [4:47:54<07:54,  4.32s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏   | 3887/3996 [4:47:58<07:46,  4.28s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏   | 3888/3996 [4:48:02<07:39,  4.26s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎   | 3889/3996 [4:48:06<07:33,  4.24s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎   | 3890/3996 [4:48:10<07:27,  4.22s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎   | 3891/3996 [4:48:14<07:22,  4.21s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍   | 3892/3996 [4:48:19<07:33,  4.36s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍   | 3893/3996 [4:48:23<07:24,  4.31s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍   | 3894/3996 [4:48:28<07:16,  4.28s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍   | 3895/3996 [4:48:32<07:09,  4.25s/it]
 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌   | 3896/3996 [4:48:36<07:03,  4.23s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌   | 3897/3996 [4:48:40<06:57,  4.22s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌   | 3898/3996 [4:48:44<06:57,  4.26s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 3899/3996 [4:48:49<07:06,  4.40s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 3900/3996 [4:48:53<06:55,  4.33s/it]
                                                                                                                                                                                      
{'loss': 0.4713, 'grad_norm': 0.21627213060855865, 'learning_rate': 3.0574115006383185e-07, 'ppl': 1.6021, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4239.36, 'total_tokens': 74606090, 'epoch': 2.93}
+
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 3900/3996 [4:48:53<06:55,  4.33s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 3901/3996 [4:48:58<06:46,  4.28s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋   | 3902/3996 [4:49:02<06:39,  4.25s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊   | 3903/3996 [4:49:06<06:33,  4.23s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊   | 3904/3996 [4:49:10<06:28,  4.22s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊   | 3905/3996 [4:49:14<06:23,  4.21s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊   | 3906/3996 [4:49:19<06:33,  4.37s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉   | 3907/3996 [4:49:23<06:24,  4.32s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉   | 3908/3996 [4:49:27<06:15,  4.27s/it]
 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉   | 3909/3996 [4:49:32<06:08,  4.24s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████   | 3910/3996 [4:49:36<06:03,  4.23s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████   | 3911/3996 [4:49:40<05:57,  4.21s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████   | 3912/3996 [4:49:44<05:52,  4.20s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████   | 3913/3996 [4:49:49<06:01,  4.36s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏  | 3914/3996 [4:49:53<05:53,  4.31s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏  | 3915/3996 [4:49:57<05:46,  4.27s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏  | 3916/3996 [4:50:02<05:46,  4.33s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎  | 3917/3996 [4:50:06<05:38,  4.29s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎  | 3918/3996 [4:50:10<05:32,  4.26s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎  | 3919/3996 [4:50:14<05:26,  4.23s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎  | 3920/3996 [4:50:19<05:33,  4.38s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 3921/3996 [4:50:23<05:24,  4.33s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 3922/3996 [4:50:27<05:16,  4.28s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 3923/3996 [4:50:32<05:09,  4.25s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 3924/3996 [4:50:36<05:04,  4.23s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌  | 3925/3996 [4:50:40<04:59,  4.22s/it]
                                                                                                                                                                                      
{'loss': 0.4893, 'grad_norm': 0.22113533318042755, 'learning_rate': 1.6849027966816532e-07, 'ppl': 1.6312, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4207.44, 'total_tokens': 75045579, 'epoch': 2.94}
+
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌  | 3925/3996 [4:50:40<04:59,  4.22s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌  | 3926/3996 [4:50:44<04:54,  4.21s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌  | 3927/3996 [4:50:49<05:01,  4.37s/it]
 98%|████████████████████████████████████████████████████████████████████████████��███████████████████████████████████████████████████████████▋  | 3928/3996 [4:50:53<04:53,  4.32s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋  | 3929/3996 [4:50:57<04:46,  4.28s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋  | 3930/3996 [4:51:01<04:40,  4.25s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋  | 3931/3996 [4:51:06<04:35,  4.23s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊  | 3932/3996 [4:51:10<04:30,  4.22s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊  | 3933/3996 [4:51:14<04:25,  4.22s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊  | 3934/3996 [4:51:19<04:30,  4.37s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉  | 3935/3996 [4:51:23<04:23,  4.31s/it]
 98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉  | 3936/3996 [4:51:27<04:16,  4.27s/it]
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉  | 3937/3996 [4:51:31<04:10,  4.25s/it]
 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉  | 3938/3996 [4:51:35<04:05,  4.23s/it]
 99%|███████████████████████████████████████████████████████████████████████████████████████████████████████████��█████████████████████████████  | 3939/3996 [4:51:40<04:00,  4.22s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████  | 3940/3996 [4:51:44<03:55,  4.21s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████  | 3941/3996 [4:51:49<03:59,  4.36s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████  | 3942/3996 [4:51:53<03:52,  4.31s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 3943/3996 [4:51:57<03:46,  4.28s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 3944/3996 [4:52:01<03:40,  4.25s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 3945/3996 [4:52:05<03:35,  4.23s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 3946/3996 [4:52:10<03:30,  4.22s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 3947/3996 [4:52:14<03:26,  4.21s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 3948/3996 [4:52:18<03:29,  4.37s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 3949/3996 [4:52:23<03:22,  4.31s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 3950/3996 [4:52:27<03:16,  4.28s/it]
                                                                                                                                                                                      
{'loss': 0.4804, 'grad_norm': 0.21696196496486664, 'learning_rate': 7.180845548145909e-08, 'ppl': 1.6167, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4152.94, 'total_tokens': 75479895, 'epoch': 2.96}
+
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 3950/3996 [4:52:27<03:16,  4.28s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 3951/3996 [4:52:31<03:11,  4.25s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 3952/3996 [4:52:35<03:06,  4.23s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 3953/3996 [4:52:39<03:01,  4.22s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 3954/3996 [4:52:44<02:57,  4.22s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 3955/3996 [4:52:48<02:59,  4.37s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 3956/3996 [4:52:53<02:52,  4.32s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 3957/3996 [4:52:57<02:47,  4.29s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 3958/3996 [4:53:01<02:41,  4.26s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 3959/3996 [4:53:05<02:36,  4.24s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 3960/3996 [4:53:09<02:31,  4.22s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 3961/3996 [4:53:14<02:27,  4.22s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 3962/3996 [4:53:18<02:28,  4.37s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 3963/3996 [4:53:22<02:22,  4.32s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 3964/3996 [4:53:27<02:17,  4.28s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 3965/3996 [4:53:31<02:11,  4.25s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 3966/3996 [4:53:35<02:07,  4.24s/it]
 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 3967/3996 [4:53:39<02:02,  4.23s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 3968/3996 [4:53:43<01:57,  4.21s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 3969/3996 [4:53:48<01:57,  4.36s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 3970/3996 [4:53:52<01:52,  4.33s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 3971/3996 [4:53:57<01:47,  4.29s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 3972/3996 [4:54:01<01:42,  4.25s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 3973/3996 [4:54:05<01:37,  4.24s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 3974/3996 [4:54:09<01:32,  4.22s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 3975/3996 [4:54:13<01:28,  4.20s/it]
                                                                                                                                                                                      
{'loss': 0.4758, 'grad_norm': 0.2239820659160614, 'learning_rate': 1.5734966595948308e-08, 'ppl': 1.6093, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'tokens_per_second_per_gpu': 4224.58, 'total_tokens': 75920694, 'epoch': 2.98}
+
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 3975/3996 [4:54:13<01:28,  4.20s/it]
 99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 3976/3996 [4:54:18<01:27,  4.36s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 3977/3996 [4:54:22<01:21,  4.31s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 3978/3996 [4:54:26<01:16,  4.27s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 3979/3996 [4:54:31<01:12,  4.24s/it]
100%|██████████████████████████████��███████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 3980/3996 [4:54:35<01:07,  4.23s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 3981/3996 [4:54:39<01:03,  4.21s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 3982/3996 [4:54:43<00:58,  4.20s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 3983/3996 [4:54:48<00:56,  4.36s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 3984/3996 [4:54:52<00:51,  4.30s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌| 3985/3996 [4:54:56<00:46,  4.27s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 3986/3996 [4:55:00<00:42,  4.24s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 3987/3996 [4:55:05<00:37,  4.22s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 3988/3996 [4:55:09<00:33,  4.21s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 3989/3996 [4:55:13<00:29,  4.20s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 3990/3996 [4:55:18<00:26,  4.36s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 3991/3996 [4:55:22<00:21,  4.31s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 3992/3996 [4:55:26<00:17,  4.27s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 3993/3996 [4:55:30<00:12,  4.24s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 3994/3996 [4:55:34<00:08,  4.22s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉| 3995/3996 [4:55:39<00:04,  4.21s/it]
100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3996/3996 [4:55:43<00:00,  4.20s/it][2025-12-29 07:45:28,450] [INFO] [axolotl.core.trainers.base._save:692] [PID:3751] Saving model checkpoint to ./outputs/luau-codellama-h200-fast/checkpoint-3996
+
                                                                                                                                                                                      
{'train_runtime': 17743.6946, 'train_samples_per_second': 1.126, 'train_steps_per_second': 0.225, 'train_loss': 0.5296457291126728, 'memory/max_active (GiB)': 25.53, 'memory/max_allocated (GiB)': 25.53, 'memory/device_reserved (GiB)': 139.06, 'epoch': 3.0}
+
100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3996/3996 [4:55:43<00:00,  4.20s/it]
100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3996/3996 [4:55:43<00:00,  4.44s/it]
+[2025-12-29 07:45:29,344] [INFO] [axolotl.train.save_trained_model:233] [PID:3751] Training completed! Saving trained model to ./outputs/luau-codellama-h200-fast.
+[2025-12-29 07:45:29,658] [INFO] [axolotl.train.save_trained_model:351] [PID:3751] Model successfully saved to ./outputs/luau-codellama-h200-fast
+
\ No newline at end of file