File size: 165,028 Bytes
a4d322c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 | [2026-04-07 09:59:43,139] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:10658] baseline 0.000GB ()
[2026-04-07 09:59:43,141] [INFO] [axolotl.cli.config.load_cfg:341] [PID:10658] config:
{
"activation_offloading": false,
"adapter": "lora",
"axolotl_config_path": "config.yaml",
"base_model": "mistralai/Ministral-3-8B-Instruct-2512-BF16",
"base_model_config": "mistralai/Ministral-3-8B-Instruct-2512-BF16",
"batch_size": 8,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_90",
"fp8": true,
"n_gpu": 1,
"n_node": 1,
"tf32": true
},
"context_parallel_size": 1,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_num_proc": 26,
"datasets": [
{
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "AI-AgentSafa/dataset",
"trust_remote_code": false,
"type": "alpaca"
}
],
"ddp": false,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"eaft_alpha": 1.0,
"eaft_k": 20,
"env_capabilities": {
"torch_version": "2.8.0"
},
"eval_batch_size": 4,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"fp16": false,
"generate_samples": false,
"generation_do_sample": true,
"generation_max_new_tokens": 50,
"generation_prompt_ratio": 0.5,
"generation_temperature": 0.7,
"gradient_accumulation_steps": 2,
"gradient_checkpointing": false,
"include_tkps": true,
"is_multimodal": true,
"layer_offloading": false,
"learning_rate": 0.0002,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": false,
"load_in_8bit": false,
"local_rank": 0,
"lora_alpha": 32,
"lora_dropout": 0.05,
"lora_embedding_kernel": true,
"lora_mlp_kernel": true,
"lora_o_kernel": true,
"lora_qkv_kernel": true,
"lora_r": 16,
"lora_target_modules": [
"q_proj",
"v_proj",
"k_proj",
"o_proj",
"gate_proj",
"down_proj",
"up_proj"
],
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "cosine",
"mean_resizing_embeddings": false,
"merge_method": "memory_efficient",
"micro_batch_size": 4,
"model_config_type": "mistral3",
"model_config_type_text": "ministral3",
"num_epochs": 5.0,
"num_generation_samples": 3,
"optimizer": "adamw_bnb_8bit",
"otel_metrics_host": "localhost",
"otel_metrics_port": 8000,
"output_dir": "/workspace/outputs/ministral8b-tsql",
"pretrain_multipack_attn": true,
"processor_config": "mistralai/Ministral-3-8B-Instruct-2512-BF16",
"profiler_steps_start": 0,
"qlora_sharded_model_loading": false,
"quantize_moe_experts": false,
"ray_num_workers": 1,
"resources_per_worker": {
"GPU": 1
},
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": false,
"save_safetensors": true,
"sequence_len": 2048,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"tf32": true,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "mistralai/Ministral-3-8B-Instruct-2512-BF16",
"tokenizer_save_jinja_files": true,
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"async_prefetch": false,
"log_completions": false,
"mask_truncated_completions": false,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"replay_buffer_size": 0,
"replay_recompute_logps": true,
"reroll_max_groups": 1,
"reroll_start_fraction": 1.0,
"reward_num_workers": 1,
"scale_rewards": true,
"skip_zero_advantage_batches": true,
"sync_ref_model": false,
"use_data_producer": false,
"use_vllm": false,
"vllm_lora_sync": false,
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"use_otel_metrics": false,
"use_ray": false,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"gpu_memory_utilization": 0.9,
"host": "0.0.0.0",
"port": 8000
},
"weight_decay": 0.0,
"world_size": 1
}
[2026-04-07 09:59:43,337] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:10658] Loaded image size: 1540 from model config
tokenizer_config.json: 0.00B [00:00, ?B/s]
tokenizer_config.json: 198kB [00:00, 23.2MB/s]
tokenizer.json: 0%| | 0.00/17.1M [00:00<?, ?B/s]
tokenizer.json: 0%| | 0.00/17.1M [00:00<?, ?B/s]
tokenizer.json: 1%|▉ | 209k/17.1M [00:00<00:16, 1.05MB/s]
tokenizer.json: 100%|██████████████████████████████████████████████████████████████████████████████| 17.1M/17.1M [00:00<00:00, 50.0MB/s]
tokenizer.json: 100%|██████████████████████████████████████████████████████████████████████████████| 17.1M/17.1M [00:00<00:00, 20.9MB/s]
special_tokens_map.json: 0.00B [00:00, ?B/s]
special_tokens_map.json: 147kB [00:00, 38.0MB/s]
chat_template.jinja: 0.00B [00:00, ?B/s]
chat_template.jinja: 7.75kB [00:00, 10.0MB/s]
[2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:10658] EOS: 2 / </s>
[2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:10658] BOS: 1 / <s>
[2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:10658] PAD: 11 / <pad>
[2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:10658] UNK: 0 / <unk>
[2026-04-07 09:59:46,947] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:480] [PID:10658] Unable to find prepared dataset in last_run_prepared/1ad236e091b2b88f65fa3ad720cbbedf
[2026-04-07 09:59:46,948] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:10658] Loading raw datasets...
[2026-04-07 09:59:46,949] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:10658] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
Downloading (incomplete total...): 0.00B [00:00, ?B/s]
Fetching 0 files: 0it [00:00, ?it/s][A
Fetching 0 files: 0it [00:00, ?it/s]
Download complete: : 0.00B [00:00, ?B/s]
Download complete: : 0.00B [00:00, ?B/s]
[2026-04-07 09:59:48,278] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:10658] Loading dataset: AI-AgentSafa/dataset with base_type: alpaca and prompt_style: None
Tokenizing Prompts (num_proc=26): 0%| | 0/229 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=26): 4%|██▎ | 9/229 [00:01<00:27, 8.04 examples/s]
Tokenizing Prompts (num_proc=26): 8%|████▍ | 18/229 [00:01<00:12, 16.50 examples/s]
Tokenizing Prompts (num_proc=26): 12%|██████▋ | 27/229 [00:01<00:08, 24.17 examples/s]
Tokenizing Prompts (num_proc=26): 16%|████████▉ | 36/229 [00:01<00:06, 30.23 examples/s]
Tokenizing Prompts (num_proc=26): 20%|███████████▏ | 45/229 [00:01<00:06, 29.22 examples/s]
Tokenizing Prompts (num_proc=26): 28%|███████████████▋ | 63/229 [00:02<00:03, 45.81 examples/s]
Tokenizing Prompts (num_proc=26): 31%|█████████████████▉ | 72/229 [00:02<00:03, 46.90 examples/s]
Tokenizing Prompts (num_proc=26): 35%|████████████████████▏ | 81/229 [00:02<00:03, 48.14 examples/s]
Tokenizing Prompts (num_proc=26): 39%|██████████████████████▍ | 90/229 [00:02<00:02, 50.50 examples/s]
Tokenizing Prompts (num_proc=26): 43%|████████████████████████▋ | 99/229 [00:02<00:02, 49.96 examples/s]
Tokenizing Prompts (num_proc=26): 47%|██████████████████████████▍ | 108/229 [00:02<00:02, 50.47 examples/s]
Tokenizing Prompts (num_proc=26): 51%|████████████████████████████▌ | 117/229 [00:03<00:02, 49.52 examples/s]
Tokenizing Prompts (num_proc=26): 59%|█████████████████████████████████ | 135/229 [00:03<00:01, 48.37 examples/s]
Tokenizing Prompts (num_proc=26): 63%|███████████████████████████████████▏ | 144/229 [00:03<00:01, 50.06 examples/s]
Tokenizing Prompts (num_proc=26): 67%|█████████████████████████████████████▍ | 153/229 [00:03<00:01, 49.89 examples/s]
Tokenizing Prompts (num_proc=26): 71%|███████████████████████████████████████▌ | 162/229 [00:04<00:01, 49.59 examples/s]
Tokenizing Prompts (num_proc=26): 75%|█████████████████████████████████████████▊ | 171/229 [00:04<00:01, 51.87 examples/s]
Tokenizing Prompts (num_proc=26): 79%|████████████████████████████████████████████ | 180/229 [00:04<00:00, 51.22 examples/s]
Tokenizing Prompts (num_proc=26): 83%|██████████████████████████████████████████████▏ | 189/229 [00:04<00:00, 51.12 examples/s]
Tokenizing Prompts (num_proc=26): 86%|████████████████████████████████████████████████▏ | 197/229 [00:04<00:00, 49.94 examples/s]
Tokenizing Prompts (num_proc=26): 93%|████████████████████████████████████████████████████ | 213/229 [00:05<00:00, 54.28 examples/s]
Tokenizing Prompts (num_proc=26): 97%|██████████████████████████████████████████████████████ | 221/229 [00:05<00:00, 51.50 examples/s]
Tokenizing Prompts (num_proc=26): 100%|████████████████████████████████████████████████████████| 229/229 [00:05<00:00, 54.16 examples/s]
Tokenizing Prompts (num_proc=26): 100%|████████████████████████████████████████████████████████| 229/229 [00:05<00:00, 40.97 examples/s]
[2026-04-07 09:59:54,148] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:10658] min_input_len: 243
[2026-04-07 09:59:54,149] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:10658] max_input_len: 1357
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 0%| | 0/229 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 4%|█▎ | 9/229 [00:00<00:10, 20.72 examples/s]
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 81%|████████████████████████▎ | 186/229 [00:00<00:00, 448.13 examples/s]
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 100%|██████████████████████████████| 229/229 [00:00<00:00, 315.58 examples/s]
Saving the dataset (0/1 shards): 0%| | 0/229 [00:00<?, ? examples/s]
Saving the dataset (0/1 shards): 100%|████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 684.83 examples/s]
Saving the dataset (1/1 shards): 100%|████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 684.83 examples/s]
Saving the dataset (1/1 shards): 100%|████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 536.51 examples/s]
[2026-04-07 09:59:55,922] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:10658] total_num_tokens: 175_917
[2026-04-07 09:59:55,934] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:10658] `total_supervised_tokens: 35_600`
[2026-04-07 09:59:55,934] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:10658] total_num_steps: 144
[2026-04-07 09:59:55,934] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:10658] Maximum number of steps set at 144
[2026-04-07 09:59:55,990] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:10658] loading tokenizer... mistralai/Ministral-3-8B-Instruct-2512-BF16
[2026-04-07 09:59:58,025] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:10658] EOS: 2 / </s>
[2026-04-07 09:59:58,026] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:10658] BOS: 1 / <s>
[2026-04-07 09:59:58,026] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:10658] PAD: 11 / <pad>
[2026-04-07 09:59:58,026] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:10658] UNK: 0 / <unk>
processor_config.json: 0%| | 0.00/976 [00:00<?, ?B/s]
processor_config.json: 100%|███████████████████████████████████████████████████████████████████████████| 976/976 [00:00<00:00, 1.98MB/s]
[2026-04-07 10:00:02,499] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:10658] Loading model
[2026-04-07 10:00:02,637] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:10658] Patched OptimState8bit for torch.compile compatibility
[2026-04-07 10:00:02,637] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:10658] Patched OptimState4bit for torch.compile compatibility
[2026-04-07 10:00:02,637] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:10658] Patched OptimStateFp8 for torch.compile compatibility
[2026-04-07 10:00:02,650] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:10658] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-04-07 10:00:02,653] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:10658] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-04-07 10:00:02,654] [WARNING] [axolotl.loaders.patch_manager._apply_self_attention_lora_patch:436] [PID:10658] Cannot patch self-attention - requires no dropout
model.safetensors.index.json: 0.00B [00:00, ?B/s]
model.safetensors.index.json: 52.7kB [00:00, 44.6MB/s]
Downloading (incomplete total...): 0.00B [00:00, ?B/s]
Fetching 4 files: 0%| | 0/4 [00:00<?, ?it/s][A
Downloading (incomplete total...): 0%| | 0.00/4.92G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/9.90G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/12.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:01<?, ?B/s]
Downloading (incomplete total...): 0%| | 193k/17.8G [00:01<2:41:15, 1.84MB/s]
Downloading (incomplete total...): 0%|▏ | 67.3M/17.8G [00:03<11:18, 26.2MB/s]
Downloading (incomplete total...): 1%|▍ | 134M/17.8G [00:10<21:56, 13.4MB/s]
Downloading (incomplete total...): 3%|█▉ | 578M/17.8G [00:12<04:35, 62.6MB/s]
Downloading (incomplete total...): 4%|██▎ | 670M/17.8G [00:13<03:46, 75.7MB/s]
Downloading (incomplete total...): 4%|██▌ | 744M/17.8G [00:13<03:34, 79.9MB/s]
Downloading (incomplete total...): 5%|██▋ | 811M/17.8G [00:16<05:22, 52.8MB/s]
Downloading (incomplete total...): 6%|███▋ | 1.10G/17.8G [00:17<02:39, 105MB/s]
Downloading (incomplete total...): 8%|████▌ | 1.36G/17.8G [00:18<01:59, 138MB/s]
Downloading (incomplete total...): 8%|████▊ | 1.43G/17.8G [00:19<02:29, 110MB/s]
Downloading (incomplete total...): 12%|███████ | 2.10G/17.8G [00:26<02:27, 107MB/s]
Downloading (incomplete total...): 13%|███████▉ | 2.37G/17.8G [00:27<02:11, 118MB/s]
Downloading (incomplete total...): 19%|███████████▏ | 3.31G/17.8G [00:27<00:53, 270MB/s]
Downloading (incomplete total...): 20%|████████████▎ | 3.66G/17.8G [00:28<00:47, 297MB/s]
Downloading (incomplete total...): 25%|███████████████▏ | 4.53G/17.8G [00:29<00:32, 403MB/s]
Downloading (incomplete total...): 27%|███████████████▉ | 4.73G/17.8G [00:30<00:31, 419MB/s]
Downloading (incomplete total...): 27%|████████████████▎ | 4.86G/17.8G [00:32<00:49, 263MB/s]
Downloading (incomplete total...): 28%|████████████████▊ | 4.98G/17.8G [00:35<01:30, 142MB/s]
Downloading (incomplete total...): 30%|██████████████████ | 5.38G/17.8G [00:36<01:01, 204MB/s]
Downloading (incomplete total...): 32%|███████████████████▏ | 5.69G/17.8G [00:37<00:49, 248MB/s]
Downloading (incomplete total...): 33%|███████████████████▋ | 5.86G/17.8G [00:37<00:41, 287MB/s]
Downloading (incomplete total...): 33%|███████████████████▉ | 5.94G/17.8G [00:41<01:48, 110MB/s]
Downloading (incomplete total...): 36%|█████████████████████▍ | 6.38G/17.8G [00:44<01:25, 134MB/s]
Downloading (incomplete total...): 37%|██████████████████████▍ | 6.67G/17.8G [00:44<01:01, 180MB/s]
Downloading (incomplete total...): 39%|███████████████████████▎ | 6.94G/17.8G [00:44<00:35, 305MB/s]
Downloading (incomplete total...): 39%|███████████████████████▋ | 7.04G/17.8G [00:44<00:33, 322MB/s]
Downloading (incomplete total...): 41%|████████████████████████▌ | 7.31G/17.8G [00:45<00:32, 326MB/s]
Downloading (incomplete total...): 43%|█████████████████████████▋ | 7.65G/17.8G [00:50<01:18, 130MB/s]
Downloading (incomplete total...): 45%|██████████████████████████▊ | 7.98G/17.8G [00:51<00:55, 177MB/s]
Downloading (incomplete total...): 46%|███████████████████████████▌ | 8.18G/17.8G [00:51<00:44, 215MB/s]
Downloading (incomplete total...): 47%|████████████████████████████▍ | 8.45G/17.8G [00:53<00:48, 195MB/s]
Downloading (incomplete total...): 48%|████████████████████████████▉ | 8.59G/17.8G [00:54<00:48, 189MB/s]
Downloading (incomplete total...): 50%|██████████████████████████████▏ | 8.99G/17.8G [00:56<00:52, 169MB/s]
Downloading (incomplete total...): 54%|████████████████████████████████▎ | 9.59G/17.8G [00:56<00:26, 314MB/s]
Downloading (incomplete total...): 61%|████████████████████████████████████▎ | 10.8G/17.8G [01:00<00:20, 347MB/s]
Downloading (incomplete total...): 61%|████████████████████████████████████▎ | 10.8G/17.8G [01:00<00:19, 359MB/s]
Downloading (incomplete total...): 68%|████████████████████████████████████████▌ | 12.1G/17.8G [01:00<00:10, 562MB/s]
Downloading (incomplete total...): 77%|██████████████████████████████████████████████▍ | 13.8G/17.8G [01:01<00:05, 756MB/s]
Downloading (incomplete total...): 86%|██████████████████████████████████████████████████▊ | 15.3G/17.8G [01:02<00:02, 1.14GB/s]
Downloading (incomplete total...): 86%|██████████████████████████████████████████████████▊ | 15.3G/17.8G [01:02<00:01, 1.64GB/s]
Downloading (incomplete total...): 91%|█████████████████████████████████████████████████████▋ | 16.2G/17.8G [01:02<00:00, 1.62GB/s]
Downloading (incomplete total...): 94%|███████████████████████████████████████████████████████▋ | 16.9G/17.8G [01:03<00:00, 1.48GB/s]
Downloading (incomplete total...): 97%|█████████████████████████████████████████████████████████▎ | 17.3G/17.8G [01:03<00:00, 1.60GB/s]
Downloading (incomplete total...): 99%|██████████████████████████████████████████████████████████▋| 17.7G/17.8G [01:03<00:00, 1.30GB/s]
Fetching 4 files: 25%|████████████████████▊ | 1/4 [01:04<03:13, 64.37s/it][A
Fetching 4 files: 100%|███████████████████████████████████████████████████████████████████████████████████| 4/4 [01:04<00:00, 16.09s/it]
Download complete: 100%|███████████████████████████████████████████████████████████████████████████| 17.8G/17.8G [01:04<00:00, 1.30GB/s]
Download complete: 100%|████████████████████████████████████████████████████████████████████████████| 17.8G/17.8G [01:04<00:00, 277MB/s]
Loading weights: 0%| | 0/531 [00:00<?, ?it/s]
Loading weights: 100%|██████████████████████████████████████████████████████████████████████████████| 531/531 [00:00<00:00, 6936.84it/s]
generation_config.json: 0%| | 0.00/131 [00:00<?, ?B/s]
generation_config.json: 100%|███████████████████████████████████████████████████████████████████████████| 131/131 [00:00<00:00, 341kB/s]
[2026-04-07 10:01:10,223] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:10658] Converting modules to torch.bfloat16
[2026-04-07 10:01:11,526] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:10658] Memory usage after model load 0.000GB ()
trainable params: 53,608,448 || all params: 8,971,634,688 || trainable%: 0.5975
[2026-04-07 10:01:12,050] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:10658] after adapters 0.000GB ()
[2026-04-07 10:01:13,944] [INFO] [axolotl.monkeypatch.lora_kernels.apply_lora_kernel_patches:437] [PID:10658] LoRA kernels: dropout=0.05 enabled
[2026-04-07 10:01:18,175] [INFO] [axolotl.train.save_initial_configs:417] [PID:10658] Pre-saving adapter config to /workspace/outputs/ministral8b-tsql...
[2026-04-07 10:01:18,176] [INFO] [axolotl.train.save_initial_configs:421] [PID:10658] Pre-saving tokenizer to /workspace/outputs/ministral8b-tsql...
[2026-04-07 10:01:18,300] [INFO] [axolotl.train.save_initial_configs:426] [PID:10658] Pre-saving model config to /workspace/outputs/ministral8b-tsql...
[2026-04-07 10:01:18,307] [INFO] [axolotl.train.save_initial_configs:430] [PID:10658] Pre-saving processor to /workspace/outputs/ministral8b-tsql...
[2026-04-07 10:01:18,428] [INFO] [axolotl.train.execute_training:222] [PID:10658] Starting trainer...
0%| | 0/144 [00:00<?, ?it/s]
1%|▋ | 1/144 [00:02<06:37, 2.78s/it]
{'loss': '0.5509', 'grad_norm': '1.325', 'learning_rate': '0', 'ppl': '1.735', 'memory/max_active (GiB)': '54.64', 'memory/max_allocated (GiB)': '54.64', 'memory/device_reserved (GiB)': '56.26', 'tokens/train_per_sec_per_gpu': '321.4', 'tokens/total': 7680, 'tokens/trainable': 1212, 'epoch': '0.03448'}
1%|▋ | 1/144 [00:02<06:37, 2.78s/it]
1%|█▍ | 2/144 [00:03<03:58, 1.68s/it]
{'loss': '0.6505', 'grad_norm': '1.584', 'learning_rate': '5e-05', 'ppl': '1.917', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '56.33', 'tokens/train_per_sec_per_gpu': '462', 'tokens/total': 14848, 'tokens/trainable': 2120, 'epoch': '0.06897'}
1%|█▍ | 2/144 [00:03<03:58, 1.68s/it]
2%|██ | 3/144 [00:04<03:24, 1.45s/it]
{'loss': '0.5635', 'grad_norm': '1.339', 'learning_rate': '0.0001', 'ppl': '1.757', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '430.1', 'tokens/total': 24064, 'tokens/trainable': 3222, 'epoch': '0.1034'}
2%|██ | 3/144 [00:04<03:24, 1.45s/it]
3%|██▊ | 4/144 [00:05<02:52, 1.23s/it]
{'loss': '0.4784', 'grad_norm': '0.5427', 'learning_rate': '0.00015', 'ppl': '1.614', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '821.4', 'tokens/total': 31232, 'tokens/trainable': 4691, 'epoch': '0.1379'}
3%|██▊ | 4/144 [00:05<02:52, 1.23s/it]
3%|███▍ | 5/144 [00:06<02:45, 1.19s/it]
{'loss': '0.4012', 'grad_norm': '0.6645', 'learning_rate': '0.0002', 'ppl': '1.494', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '432.4', 'tokens/total': 40192, 'tokens/trainable': 5977, 'epoch': '0.1724'}
3%|███▍ | 5/144 [00:06<02:45, 1.19s/it]
4%|████▏ | 6/144 [00:07<02:35, 1.12s/it]
{'loss': '0.4014', 'grad_norm': '0.6378', 'learning_rate': '0.0002', 'ppl': '1.494', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '947.2', 'tokens/total': 48128, 'tokens/trainable': 7464, 'epoch': '0.2069'}
4%|████▏ | 6/144 [00:07<02:35, 1.12s/it]
5%|████▊ | 7/144 [00:08<02:31, 1.10s/it]
{'loss': '0.2771', 'grad_norm': '0.5951', 'learning_rate': '0.0001999', 'ppl': '1.319', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '609.4', 'tokens/total': 56576, 'tokens/trainable': 8589, 'epoch': '0.2414'}
5%|████▊ | 7/144 [00:08<02:31, 1.10s/it]
6%|█████▌ | 8/144 [00:09<02:25, 1.07s/it]
{'loss': '0.357', 'grad_norm': '0.6578', 'learning_rate': '0.0001998', 'ppl': '1.429', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '660.4', 'tokens/total': 64256, 'tokens/trainable': 9686, 'epoch': '0.2759'}
6%|█████▌ | 8/144 [00:09<02:25, 1.07s/it]
6%|██████▏ | 9/144 [00:11<02:28, 1.10s/it]
{'loss': '0.2799', 'grad_norm': '0.4899', 'learning_rate': '0.0001996', 'ppl': '1.323', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '56.35', 'tokens/train_per_sec_per_gpu': '504.1', 'tokens/total': 73472, 'tokens/trainable': 11052, 'epoch': '0.3103'}
6%|██████▏ | 9/144 [00:11<02:28, 1.10s/it]
7%|██████▊ | 10/144 [00:11<02:15, 1.01s/it]
{'loss': '0.3391', 'grad_norm': '0.639', 'learning_rate': '0.0001994', 'ppl': '1.404', 'memory/max_active (GiB)': '42.73', 'memory/max_allocated (GiB)': '42.73', 'memory/device_reserved (GiB)': '56.35', 'tokens/train_per_sec_per_gpu': '560.5', 'tokens/total': 79872, 'tokens/trainable': 11894, 'epoch': '0.3448'}
7%|██████▊ | 10/144 [00:11<02:15, 1.01s/it]
8%|███████▍ | 11/144 [00:13<02:19, 1.05s/it]
{'loss': '0.2332', 'grad_norm': '0.4743', 'learning_rate': '0.0001991', 'ppl': '1.263', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '880.7', 'tokens/total': 89088, 'tokens/trainable': 13476, 'epoch': '0.3793'}
8%|███████▍ | 11/144 [00:13<02:19, 1.05s/it]
8%|████████▏ | 12/144 [00:13<02:13, 1.01s/it]
{'loss': '0.3007', 'grad_norm': '0.5442', 'learning_rate': '0.0001988', 'ppl': '1.351', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '333', 'tokens/total': 96256, 'tokens/trainable': 14346, 'epoch': '0.4138'}
8%|████████▏ | 12/144 [00:13<02:13, 1.01s/it]
9%|████████▊ | 13/144 [00:15<02:13, 1.02s/it]
{'loss': '0.3264', 'grad_norm': '0.4993', 'learning_rate': '0.0001984', 'ppl': '1.386', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '544.6', 'tokens/total': 104448, 'tokens/trainable': 15521, 'epoch': '0.4483'}
9%|████████▊ | 13/144 [00:15<02:13, 1.02s/it]
10%|█████████▌ | 14/144 [00:16<02:21, 1.09s/it]
{'loss': '0.3184', 'grad_norm': '0.4458', 'learning_rate': '0.000198', 'ppl': '1.375', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '778.9', 'tokens/total': 114432, 'tokens/trainable': 17296, 'epoch': '0.4828'}
10%|█████████▌ | 14/144 [00:16<02:21, 1.09s/it]
10%|██████████▏ | 15/144 [00:17<02:19, 1.08s/it]
{'loss': '0.2615', 'grad_norm': '0.5099', 'learning_rate': '0.0001975', 'ppl': '1.299', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '564.3', 'tokens/total': 122880, 'tokens/trainable': 18791, 'epoch': '0.5172'}
10%|██████████▏ | 15/144 [00:17<02:19, 1.08s/it]
11%|██████████▉ | 16/144 [00:18<02:17, 1.07s/it]
{'loss': '0.2764', 'grad_norm': '0.4578', 'learning_rate': '0.000197', 'ppl': '1.318', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '486.4', 'tokens/total': 131328, 'tokens/trainable': 19982, 'epoch': '0.5517'}
11%|██████████▉ | 16/144 [00:18<02:17, 1.07s/it]
12%|███████████▌ | 17/144 [00:19<02:16, 1.08s/it]
{'loss': '0.2322', 'grad_norm': '0.9374', 'learning_rate': '0.0001964', 'ppl': '1.261', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '388.8', 'tokens/total': 140032, 'tokens/trainable': 21078, 'epoch': '0.5862'}
12%|███████████▌ | 17/144 [00:19<02:16, 1.08s/it]
12%|████████████▎ | 18/144 [00:20<02:07, 1.01s/it]
{'loss': '0.2012', 'grad_norm': '0.5918', 'learning_rate': '0.0001958', 'ppl': '1.223', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '364.5', 'tokens/total': 146944, 'tokens/trainable': 21998, 'epoch': '0.6207'}
12%|████████████▎ | 18/144 [00:20<02:07, 1.01s/it]
13%|████████████▉ | 19/144 [00:21<01:59, 1.04it/s]
{'loss': '0.3241', 'grad_norm': '0.5199', 'learning_rate': '0.0001951', 'ppl': '1.383', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '546.3', 'tokens/total': 153600, 'tokens/trainable': 23122, 'epoch': '0.6552'}
13%|████████████▉ | 19/144 [00:21<01:59, 1.04it/s]
14%|█████████████▌ | 20/144 [00:22<02:04, 1.00s/it]
{'loss': '0.2049', 'grad_norm': '0.4991', 'learning_rate': '0.0001944', 'ppl': '1.227', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '583.8', 'tokens/total': 162304, 'tokens/trainable': 24462, 'epoch': '0.6897'}
14%|█████████████▌ | 20/144 [00:22<02:04, 1.00s/it]
15%|██████████████▎ | 21/144 [00:23<02:05, 1.02s/it]
{'loss': '0.3145', 'grad_norm': '0.4764', 'learning_rate': '0.0001936', 'ppl': '1.37', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '1197', 'tokens/total': 170752, 'tokens/trainable': 26354, 'epoch': '0.7241'}
15%|██████████████▎ | 21/144 [00:23<02:05, 1.02s/it]
15%|██████████████▉ | 22/144 [00:24<02:02, 1.01s/it]
{'loss': '0.2909', 'grad_norm': '0.5218', 'learning_rate': '0.0001928', 'ppl': '1.338', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '343.8', 'tokens/total': 178432, 'tokens/trainable': 27242, 'epoch': '0.7586'}
15%|██████████████▉ | 22/144 [00:24<02:02, 1.01s/it]
16%|███████████████▋ | 23/144 [00:25<01:59, 1.01it/s]
{'loss': '0.38', 'grad_norm': '0.5654', 'learning_rate': '0.000192', 'ppl': '1.462', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '409.6', 'tokens/total': 185856, 'tokens/trainable': 28259, 'epoch': '0.7931'}
16%|███████████████▋ | 23/144 [00:25<01:59, 1.01it/s]
17%|████████████████▎ | 24/144 [00:26<02:00, 1.01s/it]
{'loss': '0.2585', 'grad_norm': '0.4793', 'learning_rate': '0.000191', 'ppl': '1.295', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '615', 'tokens/total': 194304, 'tokens/trainable': 29842, 'epoch': '0.8276'}
17%|████████████████▎ | 24/144 [00:26<02:00, 1.01s/it]
17%|█████████████████ | 25/144 [00:27<02:04, 1.04s/it]
{'loss': '0.2904', 'grad_norm': '0.4865', 'learning_rate': '0.0001901', 'ppl': '1.337', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '641.3', 'tokens/total': 203264, 'tokens/trainable': 31205, 'epoch': '0.8621'}
17%|█████████████████ | 25/144 [00:27<02:04, 1.04s/it]
18%|█████████████████▋ | 26/144 [00:28<02:01, 1.03s/it]
{'loss': '0.2391', 'grad_norm': '0.601', 'learning_rate': '0.0001891', 'ppl': '1.27', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '482.1', 'tokens/total': 211200, 'tokens/trainable': 32132, 'epoch': '0.8966'}
18%|█████████████████▋ | 26/144 [00:28<02:01, 1.03s/it]
19%|██████████████████▍ | 27/144 [00:29<02:06, 1.09s/it]
{'loss': '0.1898', 'grad_norm': '0.5887', 'learning_rate': '0.0001881', 'ppl': '1.209', 'memory/max_active (GiB)': '60.88', 'memory/max_allocated (GiB)': '60.88', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '739.7', 'tokens/total': 220928, 'tokens/trainable': 33377, 'epoch': '0.931'}
19%|██████████████████▍ | 27/144 [00:29<02:06, 1.09s/it]
19%|███████████████████ | 28/144 [00:30<02:05, 1.08s/it]
{'loss': '0.2886', 'grad_norm': '0.4934', 'learning_rate': '0.000187', 'ppl': '1.335', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '611', 'tokens/total': 229632, 'tokens/trainable': 34774, 'epoch': '0.9655'}
19%|███████████████████ | 28/144 [00:30<02:05, 1.08s/it]
20%|███████████████████▋ | 29/144 [00:31<01:53, 1.01it/s]
{'loss': '0.3665', 'grad_norm': '0.7276', 'learning_rate': '0.0001858', 'ppl': '1.443', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '524.1', 'tokens/total': 234496, 'tokens/trainable': 35600, 'epoch': '1'}
20%|███████████████████▋ | 29/144 [00:31<01:53, 1.01it/s][2026-04-07 10:01:50,299] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-29
21%|████████████████████▍ | 30/144 [00:33<02:33, 1.35s/it]
{'loss': '0.1355', 'grad_norm': '0.3634', 'learning_rate': '0.0001847', 'ppl': '1.145', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '506', 'tokens/total': 242944, 'tokens/trainable': 36619, 'epoch': '1.034'}
21%|████████████████████▍ | 30/144 [00:33<02:33, 1.35s/it]
22%|█████████████████████ | 31/144 [00:34<02:16, 1.21s/it]
{'loss': '0.1936', 'grad_norm': '0.381', 'learning_rate': '0.0001835', 'ppl': '1.214', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '489.8', 'tokens/total': 250112, 'tokens/trainable': 37893, 'epoch': '1.069'}
22%|█████████████████████ | 31/144 [00:34<02:16, 1.21s/it]
22%|█████████████████████▊ | 32/144 [00:35<02:10, 1.17s/it]
{'loss': '0.1513', 'grad_norm': '0.3618', 'learning_rate': '0.0001822', 'ppl': '1.163', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '593.3', 'tokens/total': 258560, 'tokens/trainable': 38962, 'epoch': '1.103'}
22%|█████████████████████▊ | 32/144 [00:35<02:10, 1.17s/it]
23%|██████████████████████▍ | 33/144 [00:36<02:05, 1.13s/it]
{'loss': '0.1309', 'grad_norm': '0.2941', 'learning_rate': '0.0001809', 'ppl': '1.14', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '250.5', 'tokens/total': 267008, 'tokens/trainable': 40116, 'epoch': '1.138'}
23%|██████████████████████▍ | 33/144 [00:36<02:05, 1.13s/it]
24%|███████████████████████▏ | 34/144 [00:37<02:00, 1.10s/it]
{'loss': '0.1406', 'grad_norm': '0.3014', 'learning_rate': '0.0001796', 'ppl': '1.151', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '639', 'tokens/total': 275200, 'tokens/trainable': 41185, 'epoch': '1.172'}
24%|███████████████████████▏ | 34/144 [00:37<02:00, 1.10s/it]
24%|███████████████████████▊ | 35/144 [00:38<01:58, 1.09s/it]
{'loss': '0.1571', 'grad_norm': '0.3438', 'learning_rate': '0.0001782', 'ppl': '1.17', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '670.3', 'tokens/total': 283904, 'tokens/trainable': 42527, 'epoch': '1.207'}
24%|███████████████████████▊ | 35/144 [00:38<01:58, 1.09s/it]
25%|████████████████████████▌ | 36/144 [00:39<01:56, 1.07s/it]
{'loss': '0.1538', 'grad_norm': '0.5088', 'learning_rate': '0.0001768', 'ppl': '1.166', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '573.5', 'tokens/total': 292096, 'tokens/trainable': 43532, 'epoch': '1.241'}
25%|████████████████████████▌ | 36/144 [00:39<01:56, 1.07s/it]
26%|█████████████████████████▏ | 37/144 [00:40<01:53, 1.06s/it]
{'loss': '0.1653', 'grad_norm': '0.3409', 'learning_rate': '0.0001753', 'ppl': '1.18', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '884.9', 'tokens/total': 300288, 'tokens/trainable': 45155, 'epoch': '1.276'}
26%|█████████████████████████▏ | 37/144 [00:40<01:53, 1.06s/it]
26%|█████████████████████████▊ | 38/144 [00:41<01:53, 1.07s/it]
{'loss': '0.1603', 'grad_norm': '0.3211', 'learning_rate': '0.0001738', 'ppl': '1.174', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '724.9', 'tokens/total': 308992, 'tokens/trainable': 46695, 'epoch': '1.31'}
26%|█████████████████████████▊ | 38/144 [00:41<01:53, 1.07s/it]
27%|██████████████████████████▌ | 39/144 [00:42<01:47, 1.03s/it]
{'loss': '0.1707', 'grad_norm': '0.4679', 'learning_rate': '0.0001723', 'ppl': '1.186', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '919.8', 'tokens/total': 316416, 'tokens/trainable': 47957, 'epoch': '1.345'}
27%|██████████████████████████▌ | 39/144 [00:42<01:47, 1.03s/it]
28%|███████████████████████████▏ | 40/144 [00:43<01:47, 1.04s/it]
{'loss': '0.1955', 'grad_norm': '0.7294', 'learning_rate': '0.0001707', 'ppl': '1.216', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '372.8', 'tokens/total': 324864, 'tokens/trainable': 48886, 'epoch': '1.379'}
28%|███████████████████████████▏ | 40/144 [00:43<01:47, 1.04s/it]
28%|███████████████████████████▉ | 41/144 [00:45<01:49, 1.07s/it]
{'loss': '0.1893', 'grad_norm': '0.5408', 'learning_rate': '0.0001691', 'ppl': '1.208', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '339.2', 'tokens/total': 334080, 'tokens/trainable': 49999, 'epoch': '1.414'}
28%|███████████████████████████▉ | 41/144 [00:45<01:49, 1.07s/it]
29%|████████████████████████████▌ | 42/144 [00:46<01:48, 1.07s/it]
{'loss': '0.1991', 'grad_norm': '0.3505', 'learning_rate': '0.0001675', 'ppl': '1.22', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '648.8', 'tokens/total': 342528, 'tokens/trainable': 51544, 'epoch': '1.448'}
29%|████████████████████████████▌ | 42/144 [00:46<01:48, 1.07s/it]
30%|█████████████████████████████▎ | 43/144 [00:47<01:50, 1.10s/it]
{'loss': '0.178', 'grad_norm': '0.2984', 'learning_rate': '0.0001658', 'ppl': '1.195', 'memory/max_active (GiB)': '60.72', 'memory/max_allocated (GiB)': '60.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '666.1', 'tokens/total': 351744, 'tokens/trainable': 53278, 'epoch': '1.483'}
30%|█████████████████████████████▎ | 43/144 [00:47<01:50, 1.10s/it]
31%|█████████████████████████████▉ | 44/144 [00:48<01:44, 1.04s/it]
{'loss': '0.1854', 'grad_norm': '0.3544', 'learning_rate': '0.0001641', 'ppl': '1.204', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '452.1', 'tokens/total': 358912, 'tokens/trainable': 54387, 'epoch': '1.517'}
31%|█████████████████████████████▉ | 44/144 [00:48<01:44, 1.04s/it]
31%|██████████████████████████████▋ | 45/144 [00:49<01:45, 1.07s/it]
{'loss': '0.2335', 'grad_norm': '0.3659', 'learning_rate': '0.0001623', 'ppl': '1.263', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '808.8', 'tokens/total': 368128, 'tokens/trainable': 56249, 'epoch': '1.552'}
31%|██████████████████████████████▋ | 45/144 [00:49<01:45, 1.07s/it]
32%|███████████████████████████████▎ | 46/144 [00:50<01:39, 1.01s/it]
{'loss': '0.2009', 'grad_norm': '0.419', 'learning_rate': '0.0001606', 'ppl': '1.223', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '415', 'tokens/total': 375040, 'tokens/trainable': 57127, 'epoch': '1.586'}
32%|███████████████████████████████▎ | 46/144 [00:50<01:39, 1.01s/it]
33%|███████████████████████████████▉ | 47/144 [00:51<01:39, 1.03s/it]
{'loss': '0.1739', 'grad_norm': '0.2942', 'learning_rate': '0.0001588', 'ppl': '1.19', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '752.7', 'tokens/total': 383488, 'tokens/trainable': 58754, 'epoch': '1.621'}
33%|███████████████████████████████▉ | 47/144 [00:51<01:39, 1.03s/it]
33%|████████████████████████████████▋ | 48/144 [00:52<01:42, 1.07s/it]
{'loss': '0.1528', 'grad_norm': '0.2635', 'learning_rate': '0.0001569', 'ppl': '1.165', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '748.9', 'tokens/total': 392704, 'tokens/trainable': 60561, 'epoch': '1.655'}
33%|████████████████████████████████▋ | 48/144 [00:52<01:42, 1.07s/it]
34%|█████████████████████████████████▎ | 49/144 [00:53<01:47, 1.13s/it]
{'loss': '0.16', 'grad_norm': '0.4549', 'learning_rate': '0.0001551', 'ppl': '1.173', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '321', 'tokens/total': 402944, 'tokens/trainable': 61388, 'epoch': '1.69'}
34%|█████████████████████████████████▎ | 49/144 [00:53<01:47, 1.13s/it]
35%|██████████████████████████████████ | 50/144 [00:54<01:41, 1.08s/it]
{'loss': '0.1489', 'grad_norm': '0.926', 'learning_rate': '0.0001532', 'ppl': '1.161', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '471.2', 'tokens/total': 410624, 'tokens/trainable': 62366, 'epoch': '1.724'}
35%|██████████████████████████████████ | 50/144 [00:54<01:41, 1.08s/it]
35%|██████████████████████████████████▋ | 51/144 [00:55<01:39, 1.07s/it]
{'loss': '0.1692', 'grad_norm': '0.347', 'learning_rate': '0.0001513', 'ppl': '1.184', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '508.2', 'tokens/total': 418816, 'tokens/trainable': 63408, 'epoch': '1.759'}
35%|██████████████████████████████████▋ | 51/144 [00:55<01:39, 1.07s/it]
36%|███████████████████████████████████▍ | 52/144 [00:56<01:35, 1.03s/it]
{'loss': '0.1617', 'grad_norm': '0.3494', 'learning_rate': '0.0001494', 'ppl': '1.175', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '384.1', 'tokens/total': 426240, 'tokens/trainable': 64418, 'epoch': '1.793'}
36%|███████████████████████████████████▍ | 52/144 [00:56<01:35, 1.03s/it]
37%|████████████████████████████████████ | 53/144 [00:57<01:31, 1.00s/it]
{'loss': '0.1595', 'grad_norm': '0.3438', 'learning_rate': '0.0001474', 'ppl': '1.173', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '607.9', 'tokens/total': 433664, 'tokens/trainable': 65428, 'epoch': '1.828'}
37%|████████████████████████████████████ | 53/144 [00:57<01:31, 1.00s/it]
38%|████████████████████████████████████▊ | 54/144 [00:58<01:30, 1.01s/it]
{'loss': '0.1711', 'grad_norm': '0.2998', 'learning_rate': '0.0001454', 'ppl': '1.187', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '744', 'tokens/total': 441856, 'tokens/trainable': 66937, 'epoch': '1.862'}
38%|████████████████████████████████████▊ | 54/144 [00:58<01:30, 1.01s/it]
38%|█████████████████████████████████████▍ | 55/144 [00:59<01:29, 1.01s/it]
{'loss': '0.1223', 'grad_norm': '0.3975', 'learning_rate': '0.0001434', 'ppl': '1.13', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '442.6', 'tokens/total': 449792, 'tokens/trainable': 68254, 'epoch': '1.897'}
38%|█████████████████████████████████████▍ | 55/144 [00:59<01:29, 1.01s/it]
39%|██████████████████████████████████████ | 56/144 [01:00<01:32, 1.06s/it]
{'loss': '0.1405', 'grad_norm': '0.3045', 'learning_rate': '0.0001414', 'ppl': '1.151', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '573.7', 'tokens/total': 459008, 'tokens/trainable': 69507, 'epoch': '1.931'}
39%|██████████████████████████████████████ | 56/144 [01:00<01:32, 1.06s/it]
40%|██████████████████████████████████████▊ | 57/144 [01:01<01:30, 1.04s/it]
{'loss': '0.1483', 'grad_norm': '0.4271', 'learning_rate': '0.0001393', 'ppl': '1.16', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '350.3', 'tokens/total': 466944, 'tokens/trainable': 70510, 'epoch': '1.966'}
40%|██████████████████████████████████████▊ | 57/144 [01:01<01:30, 1.04s/it]
40%|███████████████████████████████████████▍ | 58/144 [01:02<01:22, 1.05it/s]
{'loss': '0.1251', 'grad_norm': '0.6547', 'learning_rate': '0.0001372', 'ppl': '1.133', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '85.42', 'tokens/total': 471488, 'tokens/trainable': 71200, 'epoch': '2'}
40%|███████████████████████████████████████▍ | 58/144 [01:02<01:22, 1.05it/s][2026-04-07 10:02:21,374] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-58
41%|████████████████████████████████████████▏ | 59/144 [01:04<01:48, 1.27s/it]
{'loss': '0.1034', 'grad_norm': '0.3701', 'learning_rate': '0.0001351', 'ppl': '1.109', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '487.1', 'tokens/total': 479168, 'tokens/trainable': 72191, 'epoch': '2.034'}
41%|████████████████████████████████████████▏ | 59/144 [01:04<01:48, 1.27s/it]
42%|████████████████████████████████████████▊ | 60/144 [01:05<01:40, 1.19s/it]
{'loss': '0.1253', 'grad_norm': '0.488', 'learning_rate': '0.000133', 'ppl': '1.133', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '708.4', 'tokens/total': 487360, 'tokens/trainable': 73421, 'epoch': '2.069'}
42%|████████████████████████████████████████▊ | 60/144 [01:05<01:40, 1.19s/it]
42%|█████████████████████████████████████████▌ | 61/144 [01:06<01:36, 1.16s/it]
{'loss': '0.1299', 'grad_norm': '0.5269', 'learning_rate': '0.0001309', 'ppl': '1.139', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '784.3', 'tokens/total': 496064, 'tokens/trainable': 74982, 'epoch': '2.103'}
42%|█████████████████████████████████████████▌ | 61/144 [01:06<01:36, 1.16s/it]
43%|██████████████████████████████████████████▏ | 62/144 [01:07<01:26, 1.06s/it]
{'loss': '0.09991', 'grad_norm': '0.3368', 'learning_rate': '0.0001288', 'ppl': '1.105', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '611', 'tokens/total': 502464, 'tokens/trainable': 75937, 'epoch': '2.138'}
43%|██████████████████████████████████████████▏ | 62/144 [01:07<01:26, 1.06s/it]
44%|██████████████████████████████████████████▉ | 63/144 [01:08<01:26, 1.07s/it]
{'loss': '0.1556', 'grad_norm': '0.5651', 'learning_rate': '0.0001266', 'ppl': '1.168', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '437', 'tokens/total': 511424, 'tokens/trainable': 77315, 'epoch': '2.172'}
44%|██████████████████████████████████████████▉ | 63/144 [01:08<01:26, 1.07s/it]
44%|███████████████████████████████████████████▌ | 64/144 [01:09<01:25, 1.07s/it]
{'loss': '0.1063', 'grad_norm': '0.4442', 'learning_rate': '0.0001244', 'ppl': '1.112', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '764.2', 'tokens/total': 520128, 'tokens/trainable': 78520, 'epoch': '2.207'}
44%|███████████████████████████████████████████▌ | 64/144 [01:09<01:25, 1.07s/it]
45%|████████████████████████████████████████████▏ | 65/144 [01:10<01:27, 1.11s/it]
{'loss': '0.1222', 'grad_norm': '0.2811', 'learning_rate': '0.0001223', 'ppl': '1.13', 'memory/max_active (GiB)': '60.72', 'memory/max_allocated (GiB)': '60.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '548.8', 'tokens/total': 529600, 'tokens/trainable': 80005, 'epoch': '2.241'}
45%|████████████████████████████████████████████▏ | 65/144 [01:10<01:27, 1.11s/it]
46%|████████████████████████████████████████████▉ | 66/144 [01:12<01:29, 1.15s/it]
{'loss': '0.1123', 'grad_norm': '0.4723', 'learning_rate': '0.0001201', 'ppl': '1.119', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '720.3', 'tokens/total': 539584, 'tokens/trainable': 81275, 'epoch': '2.276'}
46%|████████████████████████████████████████████▉ | 66/144 [01:12<01:29, 1.15s/it]
47%|█████████████████████████████████████████████▌ | 67/144 [01:13<01:24, 1.10s/it]
{'loss': '0.09161', 'grad_norm': '0.4366', 'learning_rate': '0.0001179', 'ppl': '1.096', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '695', 'tokens/total': 547264, 'tokens/trainable': 82351, 'epoch': '2.31'}
47%|█████████████████████████████████████████████▌ | 67/144 [01:13<01:24, 1.10s/it]
47%|██████████████████████████████████████████████▎ | 68/144 [01:14<01:24, 1.11s/it]
{'loss': '0.09943', 'grad_norm': '0.3944', 'learning_rate': '0.0001156', 'ppl': '1.105', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '696.3', 'tokens/total': 556224, 'tokens/trainable': 83649, 'epoch': '2.345'}
47%|██████████████████████████████████████████████▎ | 68/144 [01:14<01:24, 1.11s/it]
48%|██████████████████████████████████████████████▉ | 69/144 [01:15<01:20, 1.07s/it]
{'loss': '0.06493', 'grad_norm': '0.2672', 'learning_rate': '0.0001134', 'ppl': '1.067', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '519', 'tokens/total': 564160, 'tokens/trainable': 84792, 'epoch': '2.379'}
48%|██████████████████████████████████████████████▉ | 69/144 [01:15<01:20, 1.07s/it]
49%|███████████████████████████████████████████████▋ | 70/144 [01:16<01:16, 1.03s/it]
{'loss': '0.08159', 'grad_norm': '0.303', 'learning_rate': '0.0001112', 'ppl': '1.085', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '696.6', 'tokens/total': 571584, 'tokens/trainable': 85952, 'epoch': '2.414'}
49%|███████████████████████████████████████████████▋ | 70/144 [01:16<01:16, 1.03s/it]
49%|████████████████████████████████████████████████▎ | 71/144 [01:17<01:14, 1.02s/it]
{'loss': '0.1159', 'grad_norm': '0.5551', 'learning_rate': '0.000109', 'ppl': '1.123', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '816.8', 'tokens/total': 579520, 'tokens/trainable': 87204, 'epoch': '2.448'}
49%|████████████████████████████████████████████████▎ | 71/144 [01:17<01:14, 1.02s/it]
50%|█████████████████████████████████████████████████ | 72/144 [01:18<01:11, 1.00it/s]
{'loss': '0.08936', 'grad_norm': '0.2856', 'learning_rate': '0.0001067', 'ppl': '1.093', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '575.2', 'tokens/total': 586944, 'tokens/trainable': 88106, 'epoch': '2.483'}
50%|█████████████████████████████████████████████████ | 72/144 [01:18<01:11, 1.00it/s]
51%|█████████████████████████████████████████████████▋ | 73/144 [01:19<01:09, 1.03it/s]
{'loss': '0.09291', 'grad_norm': '0.365', 'learning_rate': '0.0001045', 'ppl': '1.097', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '776.9', 'tokens/total': 594112, 'tokens/trainable': 89224, 'epoch': '2.517'}
51%|█████████████████████████████████████████████████▋ | 73/144 [01:19<01:09, 1.03it/s]
51%|██████████████████████████████████████████████████▎ | 74/144 [01:20<01:11, 1.01s/it]
{'loss': '0.08388', 'grad_norm': '0.3015', 'learning_rate': '0.0001022', 'ppl': '1.087', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '1145', 'tokens/total': 602816, 'tokens/trainable': 91006, 'epoch': '2.552'}
51%|██████████████████████████████████████████████████▎ | 74/144 [01:20<01:11, 1.01s/it]
52%|███████████████████████████████████████████████████ | 75/144 [01:21<01:10, 1.02s/it]
{'loss': '0.09955', 'grad_norm': '0.5344', 'learning_rate': '0.0001', 'ppl': '1.105', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '474.3', 'tokens/total': 611008, 'tokens/trainable': 92185, 'epoch': '2.586'}
52%|███████████████████████████████████████████████████ | 75/144 [01:21<01:10, 1.02s/it]
53%|███████████████████████████████████████████████████▋ | 76/144 [01:22<01:12, 1.07s/it]
{'loss': '0.0959', 'grad_norm': '0.3283', 'learning_rate': '9.776e-05', 'ppl': '1.101', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '578.4', 'tokens/total': 620224, 'tokens/trainable': 93907, 'epoch': '2.621'}
53%|███████████████████████████████████████████████████▋ | 76/144 [01:22<01:12, 1.07s/it]
53%|████████████████████████████████████████████████████▍ | 77/144 [01:23<01:12, 1.08s/it]
{'loss': '0.08367', 'grad_norm': '0.3295', 'learning_rate': '9.551e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '599.6', 'tokens/total': 628928, 'tokens/trainable': 95437, 'epoch': '2.655'}
53%|████████████████████████████████████████████████████▍ | 77/144 [01:23<01:12, 1.08s/it]
54%|█████████████████████████████████████████████████████ | 78/144 [01:24<01:12, 1.10s/it]
{'loss': '0.0872', 'grad_norm': '0.3164', 'learning_rate': '9.327e-05', 'ppl': '1.091', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '446.9', 'tokens/total': 637888, 'tokens/trainable': 96555, 'epoch': '2.69'}
54%|█████████████████████████████████████████████████████ | 78/144 [01:24<01:12, 1.10s/it]
55%|█████████████████████████████████████████████████████▊ | 79/144 [01:25<01:08, 1.05s/it]
{'loss': '0.09911', 'grad_norm': '0.3014', 'learning_rate': '9.104e-05', 'ppl': '1.104', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '674.6', 'tokens/total': 645312, 'tokens/trainable': 97790, 'epoch': '2.724'}
55%|█████████████████████████████████████████████████████▊ | 79/144 [01:25<01:08, 1.05s/it]
56%|██████████████████████████████████████████████████████▍ | 80/144 [01:26<01:04, 1.01s/it]
{'loss': '0.115', 'grad_norm': '0.3922', 'learning_rate': '8.88e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '422.2', 'tokens/total': 652736, 'tokens/trainable': 98772, 'epoch': '2.759'}
56%|██████████████████████████████████████████████████████▍ | 80/144 [01:26<01:04, 1.01s/it]
56%|███████████████████████████████████████████████████████▏ | 81/144 [01:27<01:06, 1.05s/it]
{'loss': '0.08908', 'grad_norm': '0.3826', 'learning_rate': '8.658e-05', 'ppl': '1.093', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '605.6', 'tokens/total': 661952, 'tokens/trainable': 100149, 'epoch': '2.793'}
56%|███████████████████████████████████████████████████████▏ | 81/144 [01:27<01:06, 1.05s/it]
57%|███████████████████████████████████████████████████████▊ | 82/144 [01:28<01:07, 1.09s/it]
{'loss': '0.08324', 'grad_norm': '0.2812', 'learning_rate': '8.436e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '589.5', 'tokens/total': 671168, 'tokens/trainable': 101656, 'epoch': '2.828'}
57%|███████████████████████████████████████████████████████▊ | 82/144 [01:28<01:07, 1.09s/it]
58%|████████████████████████████████████████████████████████▍ | 83/144 [01:29<01:05, 1.07s/it]
{'loss': '0.1029', 'grad_norm': '0.3454', 'learning_rate': '8.214e-05', 'ppl': '1.108', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '910.7', 'tokens/total': 679360, 'tokens/trainable': 103158, 'epoch': '2.862'}
58%|████████████████████████████████████████████████████████▍ | 83/144 [01:29<01:05, 1.07s/it]
58%|█████████████████████████████████████████████████████████▏ | 84/144 [01:30<01:06, 1.11s/it]
{'loss': '0.09776', 'grad_norm': '0.3838', 'learning_rate': '7.994e-05', 'ppl': '1.103', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '563.4', 'tokens/total': 688832, 'tokens/trainable': 104296, 'epoch': '2.897'}
58%|█████████████████████████████████████████████████████████▏ | 84/144 [01:30<01:06, 1.11s/it]
59%|█████████████████████████████████████████████████████████▊ | 85/144 [01:31<01:00, 1.02s/it]
{'loss': '0.09194', 'grad_norm': '0.3443', 'learning_rate': '7.775e-05', 'ppl': '1.096', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '842.7', 'tokens/total': 695488, 'tokens/trainable': 105395, 'epoch': '2.931'}
59%|█████████████████████████████████████████████████████████▊ | 85/144 [01:31<01:00, 1.02s/it]
60%|██████████████████████████████████████████████████████████▌ | 86/144 [01:32<00:56, 1.02it/s]
{'loss': '0.06865', 'grad_norm': '0.3011', 'learning_rate': '7.557e-05', 'ppl': '1.071', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '333.2', 'tokens/total': 702656, 'tokens/trainable': 106404, 'epoch': '2.966'}
60%|██████████████████████████████████████████████████████████▌ | 86/144 [01:32<00:56, 1.02it/s]
60%|███████████████████████████████████████████████████████████▏ | 87/144 [01:33<00:49, 1.14it/s]
{'loss': '0.09969', 'grad_norm': '0.5199', 'learning_rate': '7.34e-05', 'ppl': '1.105', 'memory/max_active (GiB)': '40.52', 'memory/max_allocated (GiB)': '40.52', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '160.2', 'tokens/total': 706304, 'tokens/trainable': 106800, 'epoch': '3'}
60%|███████████████████████████████████████████████████████████▏ | 87/144 [01:33<00:49, 1.14it/s][2026-04-07 10:02:52,144] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-87
61%|███████████████████████████████████████████████████████████▉ | 88/144 [01:35<01:09, 1.25s/it]
{'loss': '0.06207', 'grad_norm': '0.2969', 'learning_rate': '7.124e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '462.3', 'tokens/total': 714240, 'tokens/trainable': 107984, 'epoch': '3.034'}
61%|███████████████████████████████████████████████████████████▉ | 88/144 [01:35<01:09, 1.25s/it]
62%|████████████████████████████████████████████████████████████▌ | 89/144 [01:36<01:05, 1.19s/it]
{'loss': '0.04656', 'grad_norm': '0.1968', 'learning_rate': '6.91e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '430.5', 'tokens/total': 722688, 'tokens/trainable': 109321, 'epoch': '3.069'}
62%|████████████████████████████████████████████████████████████▌ | 89/144 [01:36<01:05, 1.19s/it]
62%|█████████████████████████████████████████████████████████████▎ | 90/144 [01:37<00:59, 1.11s/it]
{'loss': '0.05607', 'grad_norm': '0.1708', 'learning_rate': '6.697e-05', 'ppl': '1.058', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '544.7', 'tokens/total': 729856, 'tokens/trainable': 110426, 'epoch': '3.103'}
62%|█████████████████████████████████████████████████████████████▎ | 90/144 [01:37<00:59, 1.11s/it]
63%|█████████████████████████████████████████████████████████████▉ | 91/144 [01:38<00:55, 1.05s/it]
{'loss': '0.06276', 'grad_norm': '0.2478', 'learning_rate': '6.486e-05', 'ppl': '1.065', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '421.2', 'tokens/total': 737024, 'tokens/trainable': 111355, 'epoch': '3.138'}
63%|█████████████████████████████████████████████████████████████▉ | 91/144 [01:38<00:55, 1.05s/it]
64%|██████████████████████████████████████████████████████████████▌ | 92/144 [01:39<00:52, 1.01s/it]
{'loss': '0.06717', 'grad_norm': '0.2365', 'learning_rate': '6.277e-05', 'ppl': '1.069', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '403.1', 'tokens/total': 744448, 'tokens/trainable': 112557, 'epoch': '3.172'}
64%|██████████████████████████████████████████████████████████████▌ | 92/144 [01:39<00:52, 1.01s/it]
65%|███████████████████████████████████████████████████████████████▎ | 93/144 [01:40<00:52, 1.02s/it]
{'loss': '0.06895', 'grad_norm': '0.2605', 'learning_rate': '6.07e-05', 'ppl': '1.071', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '704.4', 'tokens/total': 752640, 'tokens/trainable': 113772, 'epoch': '3.207'}
65%|███████████████████████████████████████████████████████████████▎ | 93/144 [01:40<00:52, 1.02s/it]
65%|███████████████████████████████████████████████████████████████▉ | 94/144 [01:41<00:54, 1.09s/it]
{'loss': '0.06202', 'grad_norm': '0.2754', 'learning_rate': '5.864e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '337.1', 'tokens/total': 762368, 'tokens/trainable': 114848, 'epoch': '3.241'}
65%|███████████████████████████████████████████████████████████████▉ | 94/144 [01:41<00:54, 1.09s/it]
66%|████████████████████████████████████████████████████████████████▋ | 95/144 [01:42<00:50, 1.03s/it]
{'loss': '0.04728', 'grad_norm': '0.3247', 'learning_rate': '5.661e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '448.3', 'tokens/total': 769536, 'tokens/trainable': 115669, 'epoch': '3.276'}
66%|████████████████████████████████████████████████████████████████▋ | 95/144 [01:42<00:50, 1.03s/it]
67%|█████████████████████████████████████████████████████████████████▎ | 96/144 [01:43<00:48, 1.02s/it]
{'loss': '0.07257', 'grad_norm': '0.3041', 'learning_rate': '5.46e-05', 'ppl': '1.075', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '554.1', 'tokens/total': 777216, 'tokens/trainable': 116758, 'epoch': '3.31'}
67%|█████████████████████████████████████████████████████████████████▎ | 96/144 [01:43<00:48, 1.02s/it]
67%|██████████████████████████████████████████████████████████████████ | 97/144 [01:44<00:48, 1.03s/it]
{'loss': '0.0398', 'grad_norm': '0.2143', 'learning_rate': '5.261e-05', 'ppl': '1.041', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '450', 'tokens/total': 785664, 'tokens/trainable': 118167, 'epoch': '3.345'}
67%|██████████████████████████████████████████████████████████████████ | 97/144 [01:44<00:48, 1.03s/it]
68%|██████████████████████████████████████████████████████████████████▋ | 98/144 [01:45<00:49, 1.07s/it]
{'loss': '0.06225', 'grad_norm': '0.2198', 'learning_rate': '5.065e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '60.88', 'memory/max_allocated (GiB)': '60.88', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '740.2', 'tokens/total': 794880, 'tokens/trainable': 119894, 'epoch': '3.379'}
68%|██████████████████████████████████████████████████████████████████▋ | 98/144 [01:45<00:49, 1.07s/it]
69%|███████████████████████████████████████████████████████████████████▍ | 99/144 [01:46<00:48, 1.09s/it]
{'loss': '0.06667', 'grad_norm': '0.3708', 'learning_rate': '4.871e-05', 'ppl': '1.069', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '1179', 'tokens/total': 804096, 'tokens/trainable': 121868, 'epoch': '3.414'}
69%|███████████████████████████████████████████████████████████████████▍ | 99/144 [01:46<00:48, 1.09s/it]
69%|███████████████████████████████████████████████████████████████████▎ | 100/144 [01:48<00:50, 1.14s/it]
{'loss': '0.05494', 'grad_norm': '0.2345', 'learning_rate': '4.68e-05', 'ppl': '1.056', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '697.1', 'tokens/total': 814336, 'tokens/trainable': 123772, 'epoch': '3.448'}
69%|███████████████████████████████████████████████████████████████████▎ | 100/144 [01:48<00:50, 1.14s/it]
70%|████████████████████████████████████████████████████████████████████ | 101/144 [01:49<00:48, 1.12s/it]
{'loss': '0.07039', 'grad_norm': '0.2905', 'learning_rate': '4.491e-05', 'ppl': '1.073', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '342.7', 'tokens/total': 822784, 'tokens/trainable': 124818, 'epoch': '3.483'}
70%|████████████████████████████████████████████████████████████████████ | 101/144 [01:49<00:48, 1.12s/it]
71%|████████████████████████████████████████████████████████████████████▋ | 102/144 [01:50<00:46, 1.10s/it]
{'loss': '0.0404', 'grad_norm': '0.2652', 'learning_rate': '4.305e-05', 'ppl': '1.041', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '587.8', 'tokens/total': 831232, 'tokens/trainable': 125777, 'epoch': '3.517'}
71%|████████████████████████████████████████████████████████████████████▋ | 102/144 [01:50<00:46, 1.10s/it]
72%|█████████████████████████████████████████████████████████████████████▍ | 103/144 [01:51<00:42, 1.05s/it]
{'loss': '0.04841', 'grad_norm': '0.3569', 'learning_rate': '4.122e-05', 'ppl': '1.05', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '476.6', 'tokens/total': 838656, 'tokens/trainable': 126543, 'epoch': '3.552'}
72%|█████████████████████████████████████████████████████████████████████▍ | 103/144 [01:51<00:42, 1.05s/it]
72%|██████████████████████████████████████████████████████████████████████ | 104/144 [01:52<00:41, 1.03s/it]
{'loss': '0.06206', 'grad_norm': '0.3093', 'learning_rate': '3.942e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '572.6', 'tokens/total': 846592, 'tokens/trainable': 127626, 'epoch': '3.586'}
72%|██████████████████████████████████████████████████████████████████████ | 104/144 [01:52<00:41, 1.03s/it]
73%|██████████████████████████████████████████████████████████████████████▋ | 105/144 [01:53<00:40, 1.03s/it]
{'loss': '0.05195', 'grad_norm': '0.2954', 'learning_rate': '3.765e-05', 'ppl': '1.053', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '587.7', 'tokens/total': 854528, 'tokens/trainable': 128875, 'epoch': '3.621'}
73%|██████████████████████████████████████████████████████████████████████▋ | 105/144 [01:53<00:40, 1.03s/it]
74%|███████████████████████████████████████████████████████████████████████▍ | 106/144 [01:54<00:38, 1.02s/it]
{'loss': '0.0636', 'grad_norm': '0.4605', 'learning_rate': '3.591e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '845.8', 'tokens/total': 862464, 'tokens/trainable': 130189, 'epoch': '3.655'}
74%|███████████████████████████████████████████████████████████████████████▍ | 106/144 [01:54<00:38, 1.02s/it]
74%|████████████████████████████████████████████████████████████████████████ | 107/144 [01:55<00:37, 1.01s/it]
{'loss': '0.05174', 'grad_norm': '0.3131', 'learning_rate': '3.421e-05', 'ppl': '1.053', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '874.8', 'tokens/total': 870400, 'tokens/trainable': 131508, 'epoch': '3.69'}
74%|████████████████████████████████████████████████████████████████████████ | 107/144 [01:55<00:37, 1.01s/it]
75%|████████████████████████████████████████████████████████████████████████▊ | 108/144 [01:56<00:38, 1.06s/it]
{'loss': '0.03808', 'grad_norm': '0.2428', 'learning_rate': '3.253e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '691.1', 'tokens/total': 879616, 'tokens/trainable': 132891, 'epoch': '3.724'}
75%|████████████████████████████████████████████████████████████████████████▊ | 108/144 [01:56<00:38, 1.06s/it]
76%|█████████████████████████████████████████████████████████████████████████▍ | 109/144 [01:57<00:39, 1.12s/it]
{'loss': '0.06047', 'grad_norm': '0.2851', 'learning_rate': '3.089e-05', 'ppl': '1.062', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '623.6', 'tokens/total': 889600, 'tokens/trainable': 134137, 'epoch': '3.759'}
76%|█████████████████████████████████████████████████████████████████████████▍ | 109/144 [01:57<00:39, 1.12s/it]
76%|██████████████████████████████████████████████████████████████████████████ | 110/144 [01:58<00:36, 1.06s/it]
{'loss': '0.06354', 'grad_norm': '0.2822', 'learning_rate': '2.929e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '577.8', 'tokens/total': 897024, 'tokens/trainable': 135094, 'epoch': '3.793'}
76%|██████████████████████████████████████████████████████████████████████████ | 110/144 [01:58<00:36, 1.06s/it]
77%|██████████████████████████████████████████████████████████████████████████▊ | 111/144 [01:59<00:35, 1.06s/it]
{'loss': '0.03931', 'grad_norm': '0.2651', 'learning_rate': '2.772e-05', 'ppl': '1.04', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '794', 'tokens/total': 905472, 'tokens/trainable': 136713, 'epoch': '3.828'}
77%|██████████████████████████████████████████████████████████████████████████▊ | 111/144 [01:59<00:35, 1.06s/it]
78%|███████████████████████████████████████████████████████████████████████████▍ | 112/144 [02:00<00:33, 1.04s/it]
{'loss': '0.0481', 'grad_norm': '0.3099', 'learning_rate': '2.619e-05', 'ppl': '1.049', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '568.8', 'tokens/total': 913152, 'tokens/trainable': 137675, 'epoch': '3.862'}
78%|███████████████████████████████████████████████████████████████████████████▍ | 112/144 [02:00<00:33, 1.04s/it]
78%|████████████████████████████████████████████████████████████████████████████ | 113/144 [02:01<00:32, 1.06s/it]
{'loss': '0.0465', 'grad_norm': '0.633', 'learning_rate': '2.469e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '933.4', 'tokens/total': 922112, 'tokens/trainable': 139199, 'epoch': '3.897'}
78%|████████████████████████████████████████████████████████████████████████████ | 113/144 [02:01<00:32, 1.06s/it]
79%|████████████████████████████████████████████████████████████████████████████▊ | 114/144 [02:02<00:31, 1.03s/it]
{'loss': '0.05563', 'grad_norm': '0.2643', 'learning_rate': '2.324e-05', 'ppl': '1.057', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '648.3', 'tokens/total': 929792, 'tokens/trainable': 140480, 'epoch': '3.931'}
79%|████████████████████████████████████████████████████████████████████████████▊ | 114/144 [02:02<00:31, 1.03s/it]
80%|█████████████████████████████████████████████████████████████████████████████▍ | 115/144 [02:03<00:28, 1.01it/s]
{'loss': '0.05304', 'grad_norm': '0.3273', 'learning_rate': '2.182e-05', 'ppl': '1.054', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '417.2', 'tokens/total': 936704, 'tokens/trainable': 141549, 'epoch': '3.966'}
80%|█████████████████████████████████████████████████████████████████████████████▍ | 115/144 [02:03<00:28, 1.01it/s]
81%|██████████████████████████████████████████████████████████████████████████████▏ | 116/144 [02:04<00:25, 1.09it/s]
{'loss': '0.0386', 'grad_norm': '0.396', 'learning_rate': '2.044e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '383.7', 'tokens/total': 941760, 'tokens/trainable': 142400, 'epoch': '4'}
81%|██████████████████████████████████████████████████████████████████████████████▏ | 116/144 [02:04<00:25, 1.09it/s][2026-04-07 10:03:23,028] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-116
81%|██████████████████████████████████████████████████████████████████████████████▊ | 117/144 [02:06<00:34, 1.26s/it]
{'loss': '0.02968', 'grad_norm': '0.1892', 'learning_rate': '1.91e-05', 'ppl': '1.03', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '352.6', 'tokens/total': 949696, 'tokens/trainable': 143669, 'epoch': '4.034'}
81%|██████████████████████████████████████████████████████████████████████████████▊ | 117/144 [02:06<00:34, 1.26s/it]
82%|███████████████████████████████████████████████████████████████████████████████▍ | 118/144 [02:07<00:31, 1.22s/it]
{'loss': '0.03344', 'grad_norm': '0.2217', 'learning_rate': '1.78e-05', 'ppl': '1.034', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '680.2', 'tokens/total': 958912, 'tokens/trainable': 145086, 'epoch': '4.069'}
82%|███████████████████████████████████████████████████████████████████████████████▍ | 118/144 [02:07<00:31, 1.22s/it]
83%|████████████████████████████████████████████████████████████████████████████████▏ | 119/144 [02:08<00:29, 1.18s/it]
{'loss': '0.03798', 'grad_norm': '0.2478', 'learning_rate': '1.654e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '397.6', 'tokens/total': 967616, 'tokens/trainable': 145921, 'epoch': '4.103'}
83%|████████████████████████████████████████████████████████████████████████████████▏ | 119/144 [02:08<00:29, 1.18s/it]
83%|████████████████████████████████████████████████████████████████████████████████▊ | 120/144 [02:09<00:27, 1.16s/it]
{'loss': '0.03804', 'grad_norm': '0.5864', 'learning_rate': '1.533e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '294.9', 'tokens/total': 976320, 'tokens/trainable': 146998, 'epoch': '4.138'}
83%|████████████████████████████████████████████████████████████████████████████████▊ | 120/144 [02:09<00:27, 1.16s/it]
84%|█████████████████████████████████████████████████████████████████████████████████▌ | 121/144 [02:10<00:27, 1.18s/it]
{'loss': '0.02537', 'grad_norm': '0.2323', 'learning_rate': '1.416e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '60.72', 'memory/max_allocated (GiB)': '60.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '704.1', 'tokens/total': 986048, 'tokens/trainable': 148666, 'epoch': '4.172'}
84%|█████████████████████████████████████████████████████████████████████████████████▌ | 121/144 [02:10<00:27, 1.18s/it]
85%|██████████████████████████████████████████████████████████████████████████████████▏ | 122/144 [02:11<00:23, 1.08s/it]
{'loss': '0.03578', 'grad_norm': '0.2683', 'learning_rate': '1.303e-05', 'ppl': '1.036', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '662.6', 'tokens/total': 992704, 'tokens/trainable': 149800, 'epoch': '4.207'}
85%|██████████████████████████████████████████████████████████████████████████████████▏ | 122/144 [02:11<00:23, 1.08s/it]
85%|██████████████████████████████████████████████████████████████████████████████████▊ | 123/144 [02:12<00:22, 1.05s/it]
{'loss': '0.03225', 'grad_norm': '0.4071', 'learning_rate': '1.194e-05', 'ppl': '1.033', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '571.1', 'tokens/total': 1000640, 'tokens/trainable': 150963, 'epoch': '4.241'}
85%|██████████████████████████████████████████████████████████████████████████████████▊ | 123/144 [02:12<00:22, 1.05s/it]
86%|███████████████████████████████████████████████████████████████████████████████████▌ | 124/144 [02:13<00:21, 1.06s/it]
{'loss': '0.02882', 'grad_norm': '0.1641', 'learning_rate': '1.09e-05', 'ppl': '1.029', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '750.5', 'tokens/total': 1009088, 'tokens/trainable': 152496, 'epoch': '4.276'}
86%|███████████████████████████████████████████████████████████████████████████████████▌ | 124/144 [02:13<00:21, 1.06s/it]
87%|████████████████████████████████████████████████████████████████████████████████████▏ | 125/144 [02:14<00:21, 1.12s/it]
{'loss': '0.03131', 'grad_norm': '0.1728', 'learning_rate': '9.903e-06', 'ppl': '1.032', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '705.2', 'tokens/total': 1019072, 'tokens/trainable': 153857, 'epoch': '4.31'}
87%|████████████████████████████████████████████████████████████████████████████████████▏ | 125/144 [02:14<00:21, 1.12s/it]
88%|████████████████████████████████████████████████████████████████████████████████████▉ | 126/144 [02:16<00:20, 1.11s/it]
{'loss': '0.02575', 'grad_norm': '0.1367', 'learning_rate': '8.952e-06', 'ppl': '1.026', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '772.3', 'tokens/total': 1027776, 'tokens/trainable': 155521, 'epoch': '4.345'}
88%|████████████████████████████████████████████████████████████████████████████████████▉ | 126/144 [02:16<00:20, 1.11s/it]
88%|█████████████████████████████████████████████████████████████████████████████████████▌ | 127/144 [02:17<00:18, 1.10s/it]
{'loss': '0.03918', 'grad_norm': '0.2347', 'learning_rate': '8.047e-06', 'ppl': '1.04', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '516.6', 'tokens/total': 1036224, 'tokens/trainable': 156435, 'epoch': '4.379'}
88%|█████████████████████████████████████████████████████████████████████████████████████▌ | 127/144 [02:17<00:18, 1.10s/it]
89%|██████████████████████████████████████████████████████████████████████████████████████▏ | 128/144 [02:18<00:17, 1.11s/it]
{'loss': '0.0305', 'grad_norm': '0.2856', 'learning_rate': '7.189e-06', 'ppl': '1.031', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '321.6', 'tokens/total': 1045184, 'tokens/trainable': 157278, 'epoch': '4.414'}
89%|██████████████████████████████████████████████████████████████████████████████████████▏ | 128/144 [02:18<00:17, 1.11s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████▉ | 129/144 [02:19<00:16, 1.12s/it]
{'loss': '0.02478', 'grad_norm': '0.1711', 'learning_rate': '6.377e-06', 'ppl': '1.025', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '659', 'tokens/total': 1054400, 'tokens/trainable': 158505, 'epoch': '4.448'}
90%|██████████████████████████████████████████████████████████████████████████████████████▉ | 129/144 [02:19<00:16, 1.12s/it]
90%|███████████████████████████████████████████████████████████████████████████████████████▌ | 130/144 [02:20<00:15, 1.11s/it]
{'loss': '0.04704', 'grad_norm': '0.2165', 'learning_rate': '5.612e-06', 'ppl': '1.048', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '617.7', 'tokens/total': 1063104, 'tokens/trainable': 159759, 'epoch': '4.483'}
90%|███████████████████████████████████████████████████████████████████████████████████████▌ | 130/144 [02:20<00:15, 1.11s/it]
91%|████████████████████████████████████████████████████████████████████████████████████████▏ | 131/144 [02:21<00:14, 1.09s/it]
{'loss': '0.02228', 'grad_norm': '0.1575', 'learning_rate': '4.894e-06', 'ppl': '1.023', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '587.2', 'tokens/total': 1071296, 'tokens/trainable': 161371, 'epoch': '4.517'}
91%|████████████████████████████████████████████████████████████████████████████████████████▏ | 131/144 [02:21<00:14, 1.09s/it]
92%|████████████████████████████████████████████████████████████████████████████████████████▉ | 132/144 [02:22<00:13, 1.08s/it]
{'loss': '0.03479', 'grad_norm': '0.1925', 'learning_rate': '4.225e-06', 'ppl': '1.035', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '793.1', 'tokens/total': 1079744, 'tokens/trainable': 162649, 'epoch': '4.552'}
92%|████████████████████████████████████████████████████████████████████████████████████████▉ | 132/144 [02:22<00:13, 1.08s/it]
92%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 133/144 [02:23<00:11, 1.05s/it]
{'loss': '0.02817', 'grad_norm': '0.1716', 'learning_rate': '3.604e-06', 'ppl': '1.029', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '590.8', 'tokens/total': 1087424, 'tokens/trainable': 163754, 'epoch': '4.586'}
92%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 133/144 [02:23<00:11, 1.05s/it]
93%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 134/144 [02:24<00:10, 1.05s/it]
{'loss': '0.03311', 'grad_norm': '0.2505', 'learning_rate': '3.031e-06', 'ppl': '1.034', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '453.8', 'tokens/total': 1095616, 'tokens/trainable': 165030, 'epoch': '4.621'}
93%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 134/144 [02:24<00:10, 1.05s/it]
94%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 135/144 [02:25<00:09, 1.05s/it]
{'loss': '0.03378', 'grad_norm': '0.2401', 'learning_rate': '2.507e-06', 'ppl': '1.034', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '618.7', 'tokens/total': 1103808, 'tokens/trainable': 166465, 'epoch': '4.655'}
94%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 135/144 [02:25<00:09, 1.05s/it]
94%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 136/144 [02:26<00:08, 1.06s/it]
{'loss': '0.03617', 'grad_norm': '0.2443', 'learning_rate': '2.032e-06', 'ppl': '1.037', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '573.1', 'tokens/total': 1112512, 'tokens/trainable': 167478, 'epoch': '4.69'}
94%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 136/144 [02:26<00:08, 1.06s/it]
95%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 137/144 [02:27<00:07, 1.04s/it]
{'loss': '0.03891', 'grad_norm': '0.2393', 'learning_rate': '1.607e-06', 'ppl': '1.04', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '841.8', 'tokens/total': 1120448, 'tokens/trainable': 168824, 'epoch': '4.724'}
95%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 137/144 [02:27<00:07, 1.04s/it]
96%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 138/144 [02:28<00:06, 1.07s/it]
{'loss': '0.02679', 'grad_norm': '0.2252', 'learning_rate': '1.231e-06', 'ppl': '1.027', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '718.7', 'tokens/total': 1129408, 'tokens/trainable': 170354, 'epoch': '4.759'}
96%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 138/144 [02:28<00:06, 1.07s/it]
97%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 139/144 [02:29<00:04, 1.01it/s]
{'loss': '0.03431', 'grad_norm': '0.2079', 'learning_rate': '9.05e-07', 'ppl': '1.035', 'memory/max_active (GiB)': '42.56', 'memory/max_allocated (GiB)': '42.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '658.7', 'tokens/total': 1135808, 'tokens/trainable': 171513, 'epoch': '4.793'}
97%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 139/144 [02:29<00:04, 1.01it/s]
97%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 140/144 [02:30<00:04, 1.02s/it]
{'loss': '0.02909', 'grad_norm': '0.1657', 'learning_rate': '6.288e-07', 'ppl': '1.03', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '753', 'tokens/total': 1144512, 'tokens/trainable': 172991, 'epoch': '4.828'}
97%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 140/144 [02:30<00:04, 1.02s/it]
98%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 141/144 [02:31<00:02, 1.01it/s]
{'loss': '0.02993', 'grad_norm': '0.1878', 'learning_rate': '4.026e-07', 'ppl': '1.03', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '438', 'tokens/total': 1151936, 'tokens/trainable': 173991, 'epoch': '4.862'}
98%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 141/144 [02:31<00:02, 1.01it/s]
99%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 142/144 [02:32<00:01, 1.04it/s]
{'loss': '0.03719', 'grad_norm': '0.3583', 'learning_rate': '2.265e-07', 'ppl': '1.038', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '742', 'tokens/total': 1159104, 'tokens/trainable': 174948, 'epoch': '4.897'}
99%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 142/144 [02:32<00:01, 1.04it/s]
99%|████████████████████████████████████████████████████████████████████████████████████████████████▎| 143/144 [02:33<00:00, 1.05it/s]
{'loss': '0.02699', 'grad_norm': '0.144', 'learning_rate': '1.007e-07', 'ppl': '1.027', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '682.9', 'tokens/total': 1166528, 'tokens/trainable': 176344, 'epoch': '4.931'}
99%|████████████████████████████████████████████████████████████████████████████████████████████████▎| 143/144 [02:33<00:00, 1.05it/s]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:34<00:00, 1.10it/s]
{'loss': '0.03761', 'grad_norm': '0.1827', 'learning_rate': '2.518e-08', 'ppl': '1.038', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '470.5', 'tokens/total': 1172928, 'tokens/trainable': 177353, 'epoch': '4.966'}
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:34<00:00, 1.10it/s][2026-04-07 10:03:53,154] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-144
{'train_runtime': '155.1', 'train_samples_per_second': '7.427', 'train_steps_per_second': '0.928', 'train_loss': '0.1373', 'memory/max_active (GiB)': '16.96', 'memory/max_allocated (GiB)': '16.96', 'memory/device_reserved (GiB)': '62.72', 'epoch': '4.966', 'tokens/train_per_sec_per_gpu': '0'}
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:35<00:00, 1.10it/s]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:35<00:00, 1.08s/it]
[2026-04-07 10:03:54,163] [INFO] [axolotl.train.save_trained_model:241] [PID:10658] Training completed! Saving trained model to /workspace/outputs/ministral8b-tsql.
[2026-04-07 10:03:54,446] [INFO] [axolotl.train.save_trained_model:355] [PID:10658] Model successfully saved to /workspace/outputs/ministral8b-tsql
|