| [2026-04-07 09:59:43,139] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:10658] baseline 0.000GB () |
| [2026-04-07 09:59:43,141] [INFO] [axolotl.cli.config.load_cfg:341] [PID:10658] config: |
| { |
| "activation_offloading": false, |
| "adapter": "lora", |
| "axolotl_config_path": "config.yaml", |
| "base_model": "mistralai/Ministral-3-8B-Instruct-2512-BF16", |
| "base_model_config": "mistralai/Ministral-3-8B-Instruct-2512-BF16", |
| "batch_size": 8, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_90", |
| "fp8": true, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "context_parallel_size": 1, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 26, |
| "datasets": [ |
| { |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "AI-AgentSafa/dataset", |
| "trust_remote_code": false, |
| "type": "alpaca" |
| } |
| ], |
| "ddp": false, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.8.0" |
| }, |
| "eval_batch_size": 4, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 2, |
| "gradient_checkpointing": false, |
| "include_tkps": true, |
| "is_multimodal": true, |
| "layer_offloading": false, |
| "learning_rate": 0.0002, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "lora_alpha": 32, |
| "lora_dropout": 0.05, |
| "lora_embedding_kernel": true, |
| "lora_mlp_kernel": true, |
| "lora_o_kernel": true, |
| "lora_qkv_kernel": true, |
| "lora_r": 16, |
| "lora_target_modules": [ |
| "q_proj", |
| "v_proj", |
| "k_proj", |
| "o_proj", |
| "gate_proj", |
| "down_proj", |
| "up_proj" |
| ], |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "cosine", |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 4, |
| "model_config_type": "mistral3", |
| "model_config_type_text": "ministral3", |
| "num_epochs": 5.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_bnb_8bit", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "/workspace/outputs/ministral8b-tsql", |
| "pretrain_multipack_attn": true, |
| "processor_config": "mistralai/Ministral-3-8B-Instruct-2512-BF16", |
| "profiler_steps_start": 0, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": false, |
| "save_safetensors": true, |
| "sequence_len": 2048, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "mistralai/Ministral-3-8B-Instruct-2512-BF16", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "weight_decay": 0.0, |
| "world_size": 1 |
| } |
| [2026-04-07 09:59:43,337] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:10658] Loaded image size: 1540 from model config |
|
tokenizer_config.json: 0.00B [00:00, ?B/s]
tokenizer_config.json: 198kB [00:00, 23.2MB/s] |
|
tokenizer.json: 0%| | 0.00/17.1M [00:00<?, ?B/s]
tokenizer.json: 0%| | 0.00/17.1M [00:00<?, ?B/s]
tokenizer.json: 1%|▉ | 209k/17.1M [00:00<00:16, 1.05MB/s]
tokenizer.json: 100%|██████████████████████████████████████████████████████████████████████████████| 17.1M/17.1M [00:00<00:00, 50.0MB/s]
tokenizer.json: 100%|██████████████████████████████████████████████████████████████████████████████| 17.1M/17.1M [00:00<00:00, 20.9MB/s] |
|
special_tokens_map.json: 0.00B [00:00, ?B/s]
special_tokens_map.json: 147kB [00:00, 38.0MB/s] |
|
chat_template.jinja: 0.00B [00:00, ?B/s]
chat_template.jinja: 7.75kB [00:00, 10.0MB/s] |
| [2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:10658] EOS: 2 / </s> |
| [2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:10658] BOS: 1 / <s> |
| [2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:10658] PAD: 11 / <pad> |
| [2026-04-07 09:59:46,945] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:10658] UNK: 0 / <unk> |
| [2026-04-07 09:59:46,947] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:480] [PID:10658] Unable to find prepared dataset in last_run_prepared/1ad236e091b2b88f65fa3ad720cbbedf |
| [2026-04-07 09:59:46,948] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:10658] Loading raw datasets... |
| [2026-04-07 09:59:46,949] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:10658] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. |
|
Downloading (incomplete total...): 0.00B [00:00, ?B/s] |
|
Fetching 0 files: 0it [00:00, ?it/s][A
Fetching 0 files: 0it [00:00, ?it/s] |
|
Download complete: : 0.00B [00:00, ?B/s]
Download complete: : 0.00B [00:00, ?B/s] |
| [2026-04-07 09:59:48,278] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:10658] Loading dataset: AI-AgentSafa/dataset with base_type: alpaca and prompt_style: None |
|
Tokenizing Prompts (num_proc=26): 0%| | 0/229 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=26): 4%|██▎ | 9/229 [00:01<00:27, 8.04 examples/s]
Tokenizing Prompts (num_proc=26): 8%|████▍ | 18/229 [00:01<00:12, 16.50 examples/s]
Tokenizing Prompts (num_proc=26): 12%|██████▋ | 27/229 [00:01<00:08, 24.17 examples/s]
Tokenizing Prompts (num_proc=26): 16%|████████▉ | 36/229 [00:01<00:06, 30.23 examples/s]
Tokenizing Prompts (num_proc=26): 20%|███████████▏ | 45/229 [00:01<00:06, 29.22 examples/s]
Tokenizing Prompts (num_proc=26): 28%|███████████████▋ | 63/229 [00:02<00:03, 45.81 examples/s]
Tokenizing Prompts (num_proc=26): 31%|█████████████████▉ | 72/229 [00:02<00:03, 46.90 examples/s]
Tokenizing Prompts (num_proc=26): 35%|████████████████████▏ | 81/229 [00:02<00:03, 48.14 examples/s]
Tokenizing Prompts (num_proc=26): 39%|██████████████████████▍ | 90/229 [00:02<00:02, 50.50 examples/s]
Tokenizing Prompts (num_proc=26): 43%|████████████████████████▋ | 99/229 [00:02<00:02, 49.96 examples/s]
Tokenizing Prompts (num_proc=26): 47%|██████████████████████████▍ | 108/229 [00:02<00:02, 50.47 examples/s]
Tokenizing Prompts (num_proc=26): 51%|████████████████████████████▌ | 117/229 [00:03<00:02, 49.52 examples/s]
Tokenizing Prompts (num_proc=26): 59%|█████████████████████████████████ | 135/229 [00:03<00:01, 48.37 examples/s]
Tokenizing Prompts (num_proc=26): 63%|███████████████████████████████████▏ | 144/229 [00:03<00:01, 50.06 examples/s]
Tokenizing Prompts (num_proc=26): 67%|█████████████████████████████████████▍ | 153/229 [00:03<00:01, 49.89 examples/s]
Tokenizing Prompts (num_proc=26): 71%|███████████████████████████████████████▌ | 162/229 [00:04<00:01, 49.59 examples/s]
Tokenizing Prompts (num_proc=26): 75%|█████████████████████████████████████████▊ | 171/229 [00:04<00:01, 51.87 examples/s]
Tokenizing Prompts (num_proc=26): 79%|████████████████████████████████████████████ | 180/229 [00:04<00:00, 51.22 examples/s]
Tokenizing Prompts (num_proc=26): 83%|██████████████████████████████████████████████▏ | 189/229 [00:04<00:00, 51.12 examples/s]
Tokenizing Prompts (num_proc=26): 86%|████████████████████████████████████████████████▏ | 197/229 [00:04<00:00, 49.94 examples/s]
Tokenizing Prompts (num_proc=26): 93%|████████████████████████████████████████████████████ | 213/229 [00:05<00:00, 54.28 examples/s]
Tokenizing Prompts (num_proc=26): 97%|██████████████████████████████████████████████████████ | 221/229 [00:05<00:00, 51.50 examples/s]
Tokenizing Prompts (num_proc=26): 100%|████████████████████████████████████████████████████████| 229/229 [00:05<00:00, 54.16 examples/s]
Tokenizing Prompts (num_proc=26): 100%|████████████████████████████████████████████████████████| 229/229 [00:05<00:00, 40.97 examples/s] |
| [2026-04-07 09:59:54,148] [INFO] [axolotl.utils.data.utils._log_dataset_stats:212] [PID:10658] min_input_len: 243 |
| [2026-04-07 09:59:54,149] [INFO] [axolotl.utils.data.utils._log_dataset_stats:213] [PID:10658] max_input_len: 1357 |
|
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 0%| | 0/229 [00:00<?, ? examples/s]
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 4%|█▎ | 9/229 [00:00<00:10, 20.72 examples/s]
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 81%|████████████████████████▎ | 186/229 [00:00<00:00, 448.13 examples/s]
Dropping Invalid Sequences (<None or >2048) (num_proc=26): 100%|██████████████████████████████| 229/229 [00:00<00:00, 315.58 examples/s] |
|
Saving the dataset (0/1 shards): 0%| | 0/229 [00:00<?, ? examples/s]
Saving the dataset (0/1 shards): 100%|████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 684.83 examples/s]
Saving the dataset (1/1 shards): 100%|████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 684.83 examples/s]
Saving the dataset (1/1 shards): 100%|████████████████████████████████████████████████████████| 229/229 [00:00<00:00, 536.51 examples/s] |
| [2026-04-07 09:59:55,922] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:10658] total_num_tokens: 175_917 |
| [2026-04-07 09:59:55,934] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:10658] `total_supervised_tokens: 35_600` |
| [2026-04-07 09:59:55,934] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:10658] total_num_steps: 144 |
| [2026-04-07 09:59:55,934] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:10658] Maximum number of steps set at 144 |
| [2026-04-07 09:59:55,990] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:10658] loading tokenizer... mistralai/Ministral-3-8B-Instruct-2512-BF16 |
| [2026-04-07 09:59:58,025] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:10658] EOS: 2 / </s> |
| [2026-04-07 09:59:58,026] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:10658] BOS: 1 / <s> |
| [2026-04-07 09:59:58,026] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:10658] PAD: 11 / <pad> |
| [2026-04-07 09:59:58,026] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:10658] UNK: 0 / <unk> |
|
processor_config.json: 0%| | 0.00/976 [00:00<?, ?B/s]
processor_config.json: 100%|███████████████████████████████████████████████████████████████████████████| 976/976 [00:00<00:00, 1.98MB/s] |
| [2026-04-07 10:00:02,499] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:10658] Loading model |
| [2026-04-07 10:00:02,637] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:10658] Patched OptimState8bit for torch.compile compatibility |
| [2026-04-07 10:00:02,637] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:10658] Patched OptimState4bit for torch.compile compatibility |
| [2026-04-07 10:00:02,637] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:10658] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-04-07 10:00:02,650] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:10658] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-04-07 10:00:02,653] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:10658] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-04-07 10:00:02,654] [WARNING] [axolotl.loaders.patch_manager._apply_self_attention_lora_patch:436] [PID:10658] Cannot patch self-attention - requires no dropout |
|
model.safetensors.index.json: 0.00B [00:00, ?B/s]
model.safetensors.index.json: 52.7kB [00:00, 44.6MB/s] |
|
Downloading (incomplete total...): 0.00B [00:00, ?B/s] |
|
Fetching 4 files: 0%| | 0/4 [00:00<?, ?it/s][A
Downloading (incomplete total...): 0%| | 0.00/4.92G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/9.90G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/12.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:00<?, ?B/s]
Downloading (incomplete total...): 0%| | 0.00/17.8G [00:01<?, ?B/s]
Downloading (incomplete total...): 0%| | 193k/17.8G [00:01<2:41:15, 1.84MB/s]
Downloading (incomplete total...): 0%|▏ | 67.3M/17.8G [00:03<11:18, 26.2MB/s]
Downloading (incomplete total...): 1%|▍ | 134M/17.8G [00:10<21:56, 13.4MB/s]
Downloading (incomplete total...): 3%|█▉ | 578M/17.8G [00:12<04:35, 62.6MB/s]
Downloading (incomplete total...): 4%|██▎ | 670M/17.8G [00:13<03:46, 75.7MB/s]
Downloading (incomplete total...): 4%|██▌ | 744M/17.8G [00:13<03:34, 79.9MB/s]
Downloading (incomplete total...): 5%|██▋ | 811M/17.8G [00:16<05:22, 52.8MB/s]
Downloading (incomplete total...): 6%|███▋ | 1.10G/17.8G [00:17<02:39, 105MB/s]
Downloading (incomplete total...): 8%|████▌ | 1.36G/17.8G [00:18<01:59, 138MB/s]
Downloading (incomplete total...): 8%|████▊ | 1.43G/17.8G [00:19<02:29, 110MB/s]
Downloading (incomplete total...): 12%|███████ | 2.10G/17.8G [00:26<02:27, 107MB/s]
Downloading (incomplete total...): 13%|███████▉ | 2.37G/17.8G [00:27<02:11, 118MB/s]
Downloading (incomplete total...): 19%|███████████▏ | 3.31G/17.8G [00:27<00:53, 270MB/s]
Downloading (incomplete total...): 20%|████████████▎ | 3.66G/17.8G [00:28<00:47, 297MB/s]
Downloading (incomplete total...): 25%|███████████████▏ | 4.53G/17.8G [00:29<00:32, 403MB/s]
Downloading (incomplete total...): 27%|███████████████▉ | 4.73G/17.8G [00:30<00:31, 419MB/s]
Downloading (incomplete total...): 27%|████████████████▎ | 4.86G/17.8G [00:32<00:49, 263MB/s]
Downloading (incomplete total...): 28%|████████████████▊ | 4.98G/17.8G [00:35<01:30, 142MB/s]
Downloading (incomplete total...): 30%|██████████████████ | 5.38G/17.8G [00:36<01:01, 204MB/s]
Downloading (incomplete total...): 32%|███████████████████▏ | 5.69G/17.8G [00:37<00:49, 248MB/s]
Downloading (incomplete total...): 33%|███████████████████▋ | 5.86G/17.8G [00:37<00:41, 287MB/s]
Downloading (incomplete total...): 33%|███████████████████▉ | 5.94G/17.8G [00:41<01:48, 110MB/s]
Downloading (incomplete total...): 36%|█████████████████████▍ | 6.38G/17.8G [00:44<01:25, 134MB/s]
Downloading (incomplete total...): 37%|██████████████████████▍ | 6.67G/17.8G [00:44<01:01, 180MB/s]
Downloading (incomplete total...): 39%|███████████████████████▎ | 6.94G/17.8G [00:44<00:35, 305MB/s]
Downloading (incomplete total...): 39%|███████████████████████▋ | 7.04G/17.8G [00:44<00:33, 322MB/s]
Downloading (incomplete total...): 41%|████████████████████████▌ | 7.31G/17.8G [00:45<00:32, 326MB/s]
Downloading (incomplete total...): 43%|█████████████████████████▋ | 7.65G/17.8G [00:50<01:18, 130MB/s]
Downloading (incomplete total...): 45%|██████████████████████████▊ | 7.98G/17.8G [00:51<00:55, 177MB/s]
Downloading (incomplete total...): 46%|███████████████████████████▌ | 8.18G/17.8G [00:51<00:44, 215MB/s]
Downloading (incomplete total...): 47%|████████████████████████████▍ | 8.45G/17.8G [00:53<00:48, 195MB/s]
Downloading (incomplete total...): 48%|████████████████████████████▉ | 8.59G/17.8G [00:54<00:48, 189MB/s]
Downloading (incomplete total...): 50%|██████████████████████████████▏ | 8.99G/17.8G [00:56<00:52, 169MB/s]
Downloading (incomplete total...): 54%|████████████████████████████████▎ | 9.59G/17.8G [00:56<00:26, 314MB/s]
Downloading (incomplete total...): 61%|████████████████████████████████████▎ | 10.8G/17.8G [01:00<00:20, 347MB/s]
Downloading (incomplete total...): 61%|████████████████████████████████████▎ | 10.8G/17.8G [01:00<00:19, 359MB/s]
Downloading (incomplete total...): 68%|████████████████████████████████████████▌ | 12.1G/17.8G [01:00<00:10, 562MB/s]
Downloading (incomplete total...): 77%|██████████████████████████████████████████████▍ | 13.8G/17.8G [01:01<00:05, 756MB/s]
Downloading (incomplete total...): 86%|██████████████████████████████████████████████████▊ | 15.3G/17.8G [01:02<00:02, 1.14GB/s]
Downloading (incomplete total...): 86%|██████████████████████████████████████████████████▊ | 15.3G/17.8G [01:02<00:01, 1.64GB/s]
Downloading (incomplete total...): 91%|█████████████████████████████████████████████████████▋ | 16.2G/17.8G [01:02<00:00, 1.62GB/s]
Downloading (incomplete total...): 94%|███████████████████████████████████████████████████████▋ | 16.9G/17.8G [01:03<00:00, 1.48GB/s]
Downloading (incomplete total...): 97%|█████████████████████████████████████████████████████████▎ | 17.3G/17.8G [01:03<00:00, 1.60GB/s]
Downloading (incomplete total...): 99%|██████████████████████████████████████████████████████████▋| 17.7G/17.8G [01:03<00:00, 1.30GB/s] |
|
Fetching 4 files: 25%|████████████████████▊ | 1/4 [01:04<03:13, 64.37s/it][A
Fetching 4 files: 100%|███████████████████████████████████████████████████████████████████████████████████| 4/4 [01:04<00:00, 16.09s/it] |
|
Download complete: 100%|███████████████████████████████████████████████████████████████████████████| 17.8G/17.8G [01:04<00:00, 1.30GB/s]
Download complete: 100%|████████████████████████████████████████████████████████████████████████████| 17.8G/17.8G [01:04<00:00, 277MB/s] |
|
Loading weights: 0%| | 0/531 [00:00<?, ?it/s]
Loading weights: 100%|██████████████████████████████████████████████████████████████████████████████| 531/531 [00:00<00:00, 6936.84it/s] |
|
generation_config.json: 0%| | 0.00/131 [00:00<?, ?B/s]
generation_config.json: 100%|███████████████████████████████████████████████████████████████████████████| 131/131 [00:00<00:00, 341kB/s] |
| [2026-04-07 10:01:10,223] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:10658] Converting modules to torch.bfloat16 |
| [2026-04-07 10:01:11,526] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:10658] Memory usage after model load 0.000GB () |
| trainable params: 53,608,448 || all params: 8,971,634,688 || trainable%: 0.5975 |
| [2026-04-07 10:01:12,050] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:10658] after adapters 0.000GB () |
| [2026-04-07 10:01:13,944] [INFO] [axolotl.monkeypatch.lora_kernels.apply_lora_kernel_patches:437] [PID:10658] LoRA kernels: dropout=0.05 enabled |
| [2026-04-07 10:01:18,175] [INFO] [axolotl.train.save_initial_configs:417] [PID:10658] Pre-saving adapter config to /workspace/outputs/ministral8b-tsql... |
| [2026-04-07 10:01:18,176] [INFO] [axolotl.train.save_initial_configs:421] [PID:10658] Pre-saving tokenizer to /workspace/outputs/ministral8b-tsql... |
| [2026-04-07 10:01:18,300] [INFO] [axolotl.train.save_initial_configs:426] [PID:10658] Pre-saving model config to /workspace/outputs/ministral8b-tsql... |
| [2026-04-07 10:01:18,307] [INFO] [axolotl.train.save_initial_configs:430] [PID:10658] Pre-saving processor to /workspace/outputs/ministral8b-tsql... |
| [2026-04-07 10:01:18,428] [INFO] [axolotl.train.execute_training:222] [PID:10658] Starting trainer... |
|
0%| | 0/144 [00:00<?, ?it/s]
1%|▋ | 1/144 [00:02<06:37, 2.78s/it]
{'loss': '0.5509', 'grad_norm': '1.325', 'learning_rate': '0', 'ppl': '1.735', 'memory/max_active (GiB)': '54.64', 'memory/max_allocated (GiB)': '54.64', 'memory/device_reserved (GiB)': '56.26', 'tokens/train_per_sec_per_gpu': '321.4', 'tokens/total': 7680, 'tokens/trainable': 1212, 'epoch': '0.03448'} |
|
1%|▋ | 1/144 [00:02<06:37, 2.78s/it]
1%|█▍ | 2/144 [00:03<03:58, 1.68s/it]
{'loss': '0.6505', 'grad_norm': '1.584', 'learning_rate': '5e-05', 'ppl': '1.917', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '56.33', 'tokens/train_per_sec_per_gpu': '462', 'tokens/total': 14848, 'tokens/trainable': 2120, 'epoch': '0.06897'} |
|
1%|█▍ | 2/144 [00:03<03:58, 1.68s/it]
2%|██ | 3/144 [00:04<03:24, 1.45s/it]
{'loss': '0.5635', 'grad_norm': '1.339', 'learning_rate': '0.0001', 'ppl': '1.757', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '430.1', 'tokens/total': 24064, 'tokens/trainable': 3222, 'epoch': '0.1034'} |
|
2%|██ | 3/144 [00:04<03:24, 1.45s/it]
3%|██▊ | 4/144 [00:05<02:52, 1.23s/it]
{'loss': '0.4784', 'grad_norm': '0.5427', 'learning_rate': '0.00015', 'ppl': '1.614', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '821.4', 'tokens/total': 31232, 'tokens/trainable': 4691, 'epoch': '0.1379'} |
|
3%|██▊ | 4/144 [00:05<02:52, 1.23s/it]
3%|███▍ | 5/144 [00:06<02:45, 1.19s/it]
{'loss': '0.4012', 'grad_norm': '0.6645', 'learning_rate': '0.0002', 'ppl': '1.494', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '432.4', 'tokens/total': 40192, 'tokens/trainable': 5977, 'epoch': '0.1724'} |
|
3%|███▍ | 5/144 [00:06<02:45, 1.19s/it]
4%|████▏ | 6/144 [00:07<02:35, 1.12s/it]
{'loss': '0.4014', 'grad_norm': '0.6378', 'learning_rate': '0.0002', 'ppl': '1.494', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '947.2', 'tokens/total': 48128, 'tokens/trainable': 7464, 'epoch': '0.2069'} |
|
4%|████▏ | 6/144 [00:07<02:35, 1.12s/it]
5%|████▊ | 7/144 [00:08<02:31, 1.10s/it]
{'loss': '0.2771', 'grad_norm': '0.5951', 'learning_rate': '0.0001999', 'ppl': '1.319', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '609.4', 'tokens/total': 56576, 'tokens/trainable': 8589, 'epoch': '0.2414'} |
|
5%|████▊ | 7/144 [00:08<02:31, 1.10s/it]
6%|█████▌ | 8/144 [00:09<02:25, 1.07s/it]
{'loss': '0.357', 'grad_norm': '0.6578', 'learning_rate': '0.0001998', 'ppl': '1.429', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '56.34', 'tokens/train_per_sec_per_gpu': '660.4', 'tokens/total': 64256, 'tokens/trainable': 9686, 'epoch': '0.2759'} |
|
6%|█████▌ | 8/144 [00:09<02:25, 1.07s/it]
6%|██████▏ | 9/144 [00:11<02:28, 1.10s/it]
{'loss': '0.2799', 'grad_norm': '0.4899', 'learning_rate': '0.0001996', 'ppl': '1.323', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '56.35', 'tokens/train_per_sec_per_gpu': '504.1', 'tokens/total': 73472, 'tokens/trainable': 11052, 'epoch': '0.3103'} |
|
6%|██████▏ | 9/144 [00:11<02:28, 1.10s/it]
7%|██████▊ | 10/144 [00:11<02:15, 1.01s/it]
{'loss': '0.3391', 'grad_norm': '0.639', 'learning_rate': '0.0001994', 'ppl': '1.404', 'memory/max_active (GiB)': '42.73', 'memory/max_allocated (GiB)': '42.73', 'memory/device_reserved (GiB)': '56.35', 'tokens/train_per_sec_per_gpu': '560.5', 'tokens/total': 79872, 'tokens/trainable': 11894, 'epoch': '0.3448'} |
|
7%|██████▊ | 10/144 [00:11<02:15, 1.01s/it]
8%|███████▍ | 11/144 [00:13<02:19, 1.05s/it]
{'loss': '0.2332', 'grad_norm': '0.4743', 'learning_rate': '0.0001991', 'ppl': '1.263', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '880.7', 'tokens/total': 89088, 'tokens/trainable': 13476, 'epoch': '0.3793'} |
|
8%|███████▍ | 11/144 [00:13<02:19, 1.05s/it]
8%|████████▏ | 12/144 [00:13<02:13, 1.01s/it]
{'loss': '0.3007', 'grad_norm': '0.5442', 'learning_rate': '0.0001988', 'ppl': '1.351', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '333', 'tokens/total': 96256, 'tokens/trainable': 14346, 'epoch': '0.4138'} |
|
8%|████████▏ | 12/144 [00:13<02:13, 1.01s/it]
9%|████████▊ | 13/144 [00:15<02:13, 1.02s/it]
{'loss': '0.3264', 'grad_norm': '0.4993', 'learning_rate': '0.0001984', 'ppl': '1.386', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '544.6', 'tokens/total': 104448, 'tokens/trainable': 15521, 'epoch': '0.4483'} |
|
9%|████████▊ | 13/144 [00:15<02:13, 1.02s/it]
10%|█████████▌ | 14/144 [00:16<02:21, 1.09s/it]
{'loss': '0.3184', 'grad_norm': '0.4458', 'learning_rate': '0.000198', 'ppl': '1.375', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '778.9', 'tokens/total': 114432, 'tokens/trainable': 17296, 'epoch': '0.4828'} |
|
10%|█████████▌ | 14/144 [00:16<02:21, 1.09s/it]
10%|██████████▏ | 15/144 [00:17<02:19, 1.08s/it]
{'loss': '0.2615', 'grad_norm': '0.5099', 'learning_rate': '0.0001975', 'ppl': '1.299', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '564.3', 'tokens/total': 122880, 'tokens/trainable': 18791, 'epoch': '0.5172'} |
|
10%|██████████▏ | 15/144 [00:17<02:19, 1.08s/it]
11%|██████████▉ | 16/144 [00:18<02:17, 1.07s/it]
{'loss': '0.2764', 'grad_norm': '0.4578', 'learning_rate': '0.000197', 'ppl': '1.318', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '486.4', 'tokens/total': 131328, 'tokens/trainable': 19982, 'epoch': '0.5517'} |
|
11%|██████████▉ | 16/144 [00:18<02:17, 1.07s/it]
12%|███████████▌ | 17/144 [00:19<02:16, 1.08s/it]
{'loss': '0.2322', 'grad_norm': '0.9374', 'learning_rate': '0.0001964', 'ppl': '1.261', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '388.8', 'tokens/total': 140032, 'tokens/trainable': 21078, 'epoch': '0.5862'} |
|
12%|███████████▌ | 17/144 [00:19<02:16, 1.08s/it]
12%|████████████▎ | 18/144 [00:20<02:07, 1.01s/it]
{'loss': '0.2012', 'grad_norm': '0.5918', 'learning_rate': '0.0001958', 'ppl': '1.223', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '364.5', 'tokens/total': 146944, 'tokens/trainable': 21998, 'epoch': '0.6207'} |
|
12%|████████████▎ | 18/144 [00:20<02:07, 1.01s/it]
13%|████████████▉ | 19/144 [00:21<01:59, 1.04it/s]
{'loss': '0.3241', 'grad_norm': '0.5199', 'learning_rate': '0.0001951', 'ppl': '1.383', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '546.3', 'tokens/total': 153600, 'tokens/trainable': 23122, 'epoch': '0.6552'} |
|
13%|████████████▉ | 19/144 [00:21<01:59, 1.04it/s]
14%|█████████████▌ | 20/144 [00:22<02:04, 1.00s/it]
{'loss': '0.2049', 'grad_norm': '0.4991', 'learning_rate': '0.0001944', 'ppl': '1.227', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.51', 'tokens/train_per_sec_per_gpu': '583.8', 'tokens/total': 162304, 'tokens/trainable': 24462, 'epoch': '0.6897'} |
|
14%|█████████████▌ | 20/144 [00:22<02:04, 1.00s/it]
15%|██████████████▎ | 21/144 [00:23<02:05, 1.02s/it]
{'loss': '0.3145', 'grad_norm': '0.4764', 'learning_rate': '0.0001936', 'ppl': '1.37', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '1197', 'tokens/total': 170752, 'tokens/trainable': 26354, 'epoch': '0.7241'} |
|
15%|██████████████▎ | 21/144 [00:23<02:05, 1.02s/it]
15%|██████████████▉ | 22/144 [00:24<02:02, 1.01s/it]
{'loss': '0.2909', 'grad_norm': '0.5218', 'learning_rate': '0.0001928', 'ppl': '1.338', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '343.8', 'tokens/total': 178432, 'tokens/trainable': 27242, 'epoch': '0.7586'} |
|
15%|██████████████▉ | 22/144 [00:24<02:02, 1.01s/it]
16%|███████████████▋ | 23/144 [00:25<01:59, 1.01it/s]
{'loss': '0.38', 'grad_norm': '0.5654', 'learning_rate': '0.000192', 'ppl': '1.462', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '409.6', 'tokens/total': 185856, 'tokens/trainable': 28259, 'epoch': '0.7931'} |
|
16%|███████████████▋ | 23/144 [00:25<01:59, 1.01it/s]
17%|████████████████▎ | 24/144 [00:26<02:00, 1.01s/it]
{'loss': '0.2585', 'grad_norm': '0.4793', 'learning_rate': '0.000191', 'ppl': '1.295', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '615', 'tokens/total': 194304, 'tokens/trainable': 29842, 'epoch': '0.8276'} |
|
17%|████████████████▎ | 24/144 [00:26<02:00, 1.01s/it]
17%|█████████████████ | 25/144 [00:27<02:04, 1.04s/it]
{'loss': '0.2904', 'grad_norm': '0.4865', 'learning_rate': '0.0001901', 'ppl': '1.337', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '641.3', 'tokens/total': 203264, 'tokens/trainable': 31205, 'epoch': '0.8621'} |
|
17%|█████████████████ | 25/144 [00:27<02:04, 1.04s/it]
18%|█████████████████▋ | 26/144 [00:28<02:01, 1.03s/it]
{'loss': '0.2391', 'grad_norm': '0.601', 'learning_rate': '0.0001891', 'ppl': '1.27', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '58.52', 'tokens/train_per_sec_per_gpu': '482.1', 'tokens/total': 211200, 'tokens/trainable': 32132, 'epoch': '0.8966'} |
|
18%|█████████████████▋ | 26/144 [00:28<02:01, 1.03s/it]
19%|██████████████████▍ | 27/144 [00:29<02:06, 1.09s/it]
{'loss': '0.1898', 'grad_norm': '0.5887', 'learning_rate': '0.0001881', 'ppl': '1.209', 'memory/max_active (GiB)': '60.88', 'memory/max_allocated (GiB)': '60.88', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '739.7', 'tokens/total': 220928, 'tokens/trainable': 33377, 'epoch': '0.931'} |
|
19%|██████████████████▍ | 27/144 [00:29<02:06, 1.09s/it]
19%|███████████████████ | 28/144 [00:30<02:05, 1.08s/it]
{'loss': '0.2886', 'grad_norm': '0.4934', 'learning_rate': '0.000187', 'ppl': '1.335', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '611', 'tokens/total': 229632, 'tokens/trainable': 34774, 'epoch': '0.9655'} |
|
19%|███████████████████ | 28/144 [00:30<02:05, 1.08s/it]
20%|███████████████████▋ | 29/144 [00:31<01:53, 1.01it/s]
{'loss': '0.3665', 'grad_norm': '0.7276', 'learning_rate': '0.0001858', 'ppl': '1.443', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '524.1', 'tokens/total': 234496, 'tokens/trainable': 35600, 'epoch': '1'} |
|
20%|███████████████████▋ | 29/144 [00:31<01:53, 1.01it/s][2026-04-07 10:01:50,299] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-29 |
|
21%|████████████████████▍ | 30/144 [00:33<02:33, 1.35s/it]
{'loss': '0.1355', 'grad_norm': '0.3634', 'learning_rate': '0.0001847', 'ppl': '1.145', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '506', 'tokens/total': 242944, 'tokens/trainable': 36619, 'epoch': '1.034'} |
|
21%|████████████████████▍ | 30/144 [00:33<02:33, 1.35s/it]
22%|█████████████████████ | 31/144 [00:34<02:16, 1.21s/it]
{'loss': '0.1936', 'grad_norm': '0.381', 'learning_rate': '0.0001835', 'ppl': '1.214', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '489.8', 'tokens/total': 250112, 'tokens/trainable': 37893, 'epoch': '1.069'} |
|
22%|█████████████████████ | 31/144 [00:34<02:16, 1.21s/it]
22%|█████████████████████▊ | 32/144 [00:35<02:10, 1.17s/it]
{'loss': '0.1513', 'grad_norm': '0.3618', 'learning_rate': '0.0001822', 'ppl': '1.163', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '593.3', 'tokens/total': 258560, 'tokens/trainable': 38962, 'epoch': '1.103'} |
|
22%|█████████████████████▊ | 32/144 [00:35<02:10, 1.17s/it]
23%|██████████████████████▍ | 33/144 [00:36<02:05, 1.13s/it]
{'loss': '0.1309', 'grad_norm': '0.2941', 'learning_rate': '0.0001809', 'ppl': '1.14', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '250.5', 'tokens/total': 267008, 'tokens/trainable': 40116, 'epoch': '1.138'} |
|
23%|██████████████████████▍ | 33/144 [00:36<02:05, 1.13s/it]
24%|███████████████████████▏ | 34/144 [00:37<02:00, 1.10s/it]
{'loss': '0.1406', 'grad_norm': '0.3014', 'learning_rate': '0.0001796', 'ppl': '1.151', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '639', 'tokens/total': 275200, 'tokens/trainable': 41185, 'epoch': '1.172'} |
|
24%|███████████████████████▏ | 34/144 [00:37<02:00, 1.10s/it]
24%|███████████████████████▊ | 35/144 [00:38<01:58, 1.09s/it]
{'loss': '0.1571', 'grad_norm': '0.3438', 'learning_rate': '0.0001782', 'ppl': '1.17', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '670.3', 'tokens/total': 283904, 'tokens/trainable': 42527, 'epoch': '1.207'} |
|
24%|███████████████████████▊ | 35/144 [00:38<01:58, 1.09s/it]
25%|████████████████████████▌ | 36/144 [00:39<01:56, 1.07s/it]
{'loss': '0.1538', 'grad_norm': '0.5088', 'learning_rate': '0.0001768', 'ppl': '1.166', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '573.5', 'tokens/total': 292096, 'tokens/trainable': 43532, 'epoch': '1.241'} |
|
25%|████████████████████████▌ | 36/144 [00:39<01:56, 1.07s/it]
26%|█████████████████████████▏ | 37/144 [00:40<01:53, 1.06s/it]
{'loss': '0.1653', 'grad_norm': '0.3409', 'learning_rate': '0.0001753', 'ppl': '1.18', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '884.9', 'tokens/total': 300288, 'tokens/trainable': 45155, 'epoch': '1.276'} |
|
26%|█████████████████████████▏ | 37/144 [00:40<01:53, 1.06s/it]
26%|█████████████████████████▊ | 38/144 [00:41<01:53, 1.07s/it]
{'loss': '0.1603', 'grad_norm': '0.3211', 'learning_rate': '0.0001738', 'ppl': '1.174', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '724.9', 'tokens/total': 308992, 'tokens/trainable': 46695, 'epoch': '1.31'} |
|
26%|█████████████████████████▊ | 38/144 [00:41<01:53, 1.07s/it]
27%|██████████████████████████▌ | 39/144 [00:42<01:47, 1.03s/it]
{'loss': '0.1707', 'grad_norm': '0.4679', 'learning_rate': '0.0001723', 'ppl': '1.186', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '919.8', 'tokens/total': 316416, 'tokens/trainable': 47957, 'epoch': '1.345'} |
|
27%|██████████████████████████▌ | 39/144 [00:42<01:47, 1.03s/it]
28%|███████████████████████████▏ | 40/144 [00:43<01:47, 1.04s/it]
{'loss': '0.1955', 'grad_norm': '0.7294', 'learning_rate': '0.0001707', 'ppl': '1.216', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '372.8', 'tokens/total': 324864, 'tokens/trainable': 48886, 'epoch': '1.379'} |
|
28%|███████████████████████████▏ | 40/144 [00:43<01:47, 1.04s/it]
28%|███████████████████████████▉ | 41/144 [00:45<01:49, 1.07s/it]
{'loss': '0.1893', 'grad_norm': '0.5408', 'learning_rate': '0.0001691', 'ppl': '1.208', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '339.2', 'tokens/total': 334080, 'tokens/trainable': 49999, 'epoch': '1.414'} |
|
28%|███████████████████████████▉ | 41/144 [00:45<01:49, 1.07s/it]
29%|████████████████████████████▌ | 42/144 [00:46<01:48, 1.07s/it]
{'loss': '0.1991', 'grad_norm': '0.3505', 'learning_rate': '0.0001675', 'ppl': '1.22', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '648.8', 'tokens/total': 342528, 'tokens/trainable': 51544, 'epoch': '1.448'} |
|
29%|████████████████████████████▌ | 42/144 [00:46<01:48, 1.07s/it]
30%|█████████████████████████████▎ | 43/144 [00:47<01:50, 1.10s/it]
{'loss': '0.178', 'grad_norm': '0.2984', 'learning_rate': '0.0001658', 'ppl': '1.195', 'memory/max_active (GiB)': '60.72', 'memory/max_allocated (GiB)': '60.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '666.1', 'tokens/total': 351744, 'tokens/trainable': 53278, 'epoch': '1.483'} |
|
30%|█████████████████████████████▎ | 43/144 [00:47<01:50, 1.10s/it]
31%|█████████████████████████████▉ | 44/144 [00:48<01:44, 1.04s/it]
{'loss': '0.1854', 'grad_norm': '0.3544', 'learning_rate': '0.0001641', 'ppl': '1.204', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '452.1', 'tokens/total': 358912, 'tokens/trainable': 54387, 'epoch': '1.517'} |
|
31%|█████████████████████████████▉ | 44/144 [00:48<01:44, 1.04s/it]
31%|██████████████████████████████▋ | 45/144 [00:49<01:45, 1.07s/it]
{'loss': '0.2335', 'grad_norm': '0.3659', 'learning_rate': '0.0001623', 'ppl': '1.263', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '808.8', 'tokens/total': 368128, 'tokens/trainable': 56249, 'epoch': '1.552'} |
|
31%|██████████████████████████████▋ | 45/144 [00:49<01:45, 1.07s/it]
32%|███████████████████████████████▎ | 46/144 [00:50<01:39, 1.01s/it]
{'loss': '0.2009', 'grad_norm': '0.419', 'learning_rate': '0.0001606', 'ppl': '1.223', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '415', 'tokens/total': 375040, 'tokens/trainable': 57127, 'epoch': '1.586'} |
|
32%|███████████████████████████████▎ | 46/144 [00:50<01:39, 1.01s/it]
33%|███████████████████████████████▉ | 47/144 [00:51<01:39, 1.03s/it]
{'loss': '0.1739', 'grad_norm': '0.2942', 'learning_rate': '0.0001588', 'ppl': '1.19', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '752.7', 'tokens/total': 383488, 'tokens/trainable': 58754, 'epoch': '1.621'} |
|
33%|███████████████████████████████▉ | 47/144 [00:51<01:39, 1.03s/it]
33%|████████████████████████████████▋ | 48/144 [00:52<01:42, 1.07s/it]
{'loss': '0.1528', 'grad_norm': '0.2635', 'learning_rate': '0.0001569', 'ppl': '1.165', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '748.9', 'tokens/total': 392704, 'tokens/trainable': 60561, 'epoch': '1.655'} |
|
33%|████████████████████████████████▋ | 48/144 [00:52<01:42, 1.07s/it]
34%|█████████████████████████████████▎ | 49/144 [00:53<01:47, 1.13s/it]
{'loss': '0.16', 'grad_norm': '0.4549', 'learning_rate': '0.0001551', 'ppl': '1.173', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '321', 'tokens/total': 402944, 'tokens/trainable': 61388, 'epoch': '1.69'} |
|
34%|█████████████████████████████████▎ | 49/144 [00:53<01:47, 1.13s/it]
35%|██████████████████████████████████ | 50/144 [00:54<01:41, 1.08s/it]
{'loss': '0.1489', 'grad_norm': '0.926', 'learning_rate': '0.0001532', 'ppl': '1.161', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '471.2', 'tokens/total': 410624, 'tokens/trainable': 62366, 'epoch': '1.724'} |
|
35%|██████████████████████████████████ | 50/144 [00:54<01:41, 1.08s/it]
35%|██████████████████████████████████▋ | 51/144 [00:55<01:39, 1.07s/it]
{'loss': '0.1692', 'grad_norm': '0.347', 'learning_rate': '0.0001513', 'ppl': '1.184', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '508.2', 'tokens/total': 418816, 'tokens/trainable': 63408, 'epoch': '1.759'} |
|
35%|██████████████████████████████████▋ | 51/144 [00:55<01:39, 1.07s/it]
36%|███████████████████████████████████▍ | 52/144 [00:56<01:35, 1.03s/it]
{'loss': '0.1617', 'grad_norm': '0.3494', 'learning_rate': '0.0001494', 'ppl': '1.175', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '384.1', 'tokens/total': 426240, 'tokens/trainable': 64418, 'epoch': '1.793'} |
|
36%|███████████████████████████████████▍ | 52/144 [00:56<01:35, 1.03s/it]
37%|████████████████████████████████████ | 53/144 [00:57<01:31, 1.00s/it]
{'loss': '0.1595', 'grad_norm': '0.3438', 'learning_rate': '0.0001474', 'ppl': '1.173', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '607.9', 'tokens/total': 433664, 'tokens/trainable': 65428, 'epoch': '1.828'} |
|
37%|████████████████████████████████████ | 53/144 [00:57<01:31, 1.00s/it]
38%|████████████████████████████████████▊ | 54/144 [00:58<01:30, 1.01s/it]
{'loss': '0.1711', 'grad_norm': '0.2998', 'learning_rate': '0.0001454', 'ppl': '1.187', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '744', 'tokens/total': 441856, 'tokens/trainable': 66937, 'epoch': '1.862'} |
|
38%|████████████████████████████████████▊ | 54/144 [00:58<01:30, 1.01s/it]
38%|█████████████████████████████████████▍ | 55/144 [00:59<01:29, 1.01s/it]
{'loss': '0.1223', 'grad_norm': '0.3975', 'learning_rate': '0.0001434', 'ppl': '1.13', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '442.6', 'tokens/total': 449792, 'tokens/trainable': 68254, 'epoch': '1.897'} |
|
38%|█████████████████████████████████████▍ | 55/144 [00:59<01:29, 1.01s/it]
39%|██████████████████████████████████████ | 56/144 [01:00<01:32, 1.06s/it]
{'loss': '0.1405', 'grad_norm': '0.3045', 'learning_rate': '0.0001414', 'ppl': '1.151', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '573.7', 'tokens/total': 459008, 'tokens/trainable': 69507, 'epoch': '1.931'} |
|
39%|██████████████████████████████████████ | 56/144 [01:00<01:32, 1.06s/it]
40%|██████████████████████████████████████▊ | 57/144 [01:01<01:30, 1.04s/it]
{'loss': '0.1483', 'grad_norm': '0.4271', 'learning_rate': '0.0001393', 'ppl': '1.16', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '350.3', 'tokens/total': 466944, 'tokens/trainable': 70510, 'epoch': '1.966'} |
|
40%|██████████████████████████████████████▊ | 57/144 [01:01<01:30, 1.04s/it]
40%|███████████████████████████████████████▍ | 58/144 [01:02<01:22, 1.05it/s]
{'loss': '0.1251', 'grad_norm': '0.6547', 'learning_rate': '0.0001372', 'ppl': '1.133', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '85.42', 'tokens/total': 471488, 'tokens/trainable': 71200, 'epoch': '2'} |
|
40%|███████████████████████████████████████▍ | 58/144 [01:02<01:22, 1.05it/s][2026-04-07 10:02:21,374] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-58 |
|
41%|████████████████████████████████████████▏ | 59/144 [01:04<01:48, 1.27s/it]
{'loss': '0.1034', 'grad_norm': '0.3701', 'learning_rate': '0.0001351', 'ppl': '1.109', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '487.1', 'tokens/total': 479168, 'tokens/trainable': 72191, 'epoch': '2.034'} |
|
41%|████████████████████████████████████████▏ | 59/144 [01:04<01:48, 1.27s/it]
42%|████████████████████████████████████████▊ | 60/144 [01:05<01:40, 1.19s/it]
{'loss': '0.1253', 'grad_norm': '0.488', 'learning_rate': '0.000133', 'ppl': '1.133', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '708.4', 'tokens/total': 487360, 'tokens/trainable': 73421, 'epoch': '2.069'} |
|
42%|████████████████████████████████████████▊ | 60/144 [01:05<01:40, 1.19s/it]
42%|█████████████████████████████████████████▌ | 61/144 [01:06<01:36, 1.16s/it]
{'loss': '0.1299', 'grad_norm': '0.5269', 'learning_rate': '0.0001309', 'ppl': '1.139', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '784.3', 'tokens/total': 496064, 'tokens/trainable': 74982, 'epoch': '2.103'} |
|
42%|█████████████████████████████████████████▌ | 61/144 [01:06<01:36, 1.16s/it]
43%|██████████████████████████████████████████▏ | 62/144 [01:07<01:26, 1.06s/it]
{'loss': '0.09991', 'grad_norm': '0.3368', 'learning_rate': '0.0001288', 'ppl': '1.105', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '611', 'tokens/total': 502464, 'tokens/trainable': 75937, 'epoch': '2.138'} |
|
43%|██████████████████████████████████████████▏ | 62/144 [01:07<01:26, 1.06s/it]
44%|██████████████████████████████████████████▉ | 63/144 [01:08<01:26, 1.07s/it]
{'loss': '0.1556', 'grad_norm': '0.5651', 'learning_rate': '0.0001266', 'ppl': '1.168', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '437', 'tokens/total': 511424, 'tokens/trainable': 77315, 'epoch': '2.172'} |
|
44%|██████████████████████████████████████████▉ | 63/144 [01:08<01:26, 1.07s/it]
44%|███████████████████████████████████████████▌ | 64/144 [01:09<01:25, 1.07s/it]
{'loss': '0.1063', 'grad_norm': '0.4442', 'learning_rate': '0.0001244', 'ppl': '1.112', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '764.2', 'tokens/total': 520128, 'tokens/trainable': 78520, 'epoch': '2.207'} |
|
44%|███████████████████████████████████████████▌ | 64/144 [01:09<01:25, 1.07s/it]
45%|████████████████████████████████████████████▏ | 65/144 [01:10<01:27, 1.11s/it]
{'loss': '0.1222', 'grad_norm': '0.2811', 'learning_rate': '0.0001223', 'ppl': '1.13', 'memory/max_active (GiB)': '60.72', 'memory/max_allocated (GiB)': '60.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '548.8', 'tokens/total': 529600, 'tokens/trainable': 80005, 'epoch': '2.241'} |
|
45%|████████████████████████████████████████████▏ | 65/144 [01:10<01:27, 1.11s/it]
46%|████████████████████████████████████████████▉ | 66/144 [01:12<01:29, 1.15s/it]
{'loss': '0.1123', 'grad_norm': '0.4723', 'learning_rate': '0.0001201', 'ppl': '1.119', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '720.3', 'tokens/total': 539584, 'tokens/trainable': 81275, 'epoch': '2.276'} |
|
46%|████████████████████████████████████████████▉ | 66/144 [01:12<01:29, 1.15s/it]
47%|█████████████████████████████████████████████▌ | 67/144 [01:13<01:24, 1.10s/it]
{'loss': '0.09161', 'grad_norm': '0.4366', 'learning_rate': '0.0001179', 'ppl': '1.096', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '695', 'tokens/total': 547264, 'tokens/trainable': 82351, 'epoch': '2.31'} |
|
47%|█████████████████████████████████████████████▌ | 67/144 [01:13<01:24, 1.10s/it]
47%|██████████████████████████████████████████████▎ | 68/144 [01:14<01:24, 1.11s/it]
{'loss': '0.09943', 'grad_norm': '0.3944', 'learning_rate': '0.0001156', 'ppl': '1.105', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '696.3', 'tokens/total': 556224, 'tokens/trainable': 83649, 'epoch': '2.345'} |
|
47%|██████████████████████████████████████████████▎ | 68/144 [01:14<01:24, 1.11s/it]
48%|██████████████████████████████████████████████▉ | 69/144 [01:15<01:20, 1.07s/it]
{'loss': '0.06493', 'grad_norm': '0.2672', 'learning_rate': '0.0001134', 'ppl': '1.067', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '519', 'tokens/total': 564160, 'tokens/trainable': 84792, 'epoch': '2.379'} |
|
48%|██████████████████████████████████████████████▉ | 69/144 [01:15<01:20, 1.07s/it]
49%|███████████████████████████████████████████████▋ | 70/144 [01:16<01:16, 1.03s/it]
{'loss': '0.08159', 'grad_norm': '0.303', 'learning_rate': '0.0001112', 'ppl': '1.085', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '696.6', 'tokens/total': 571584, 'tokens/trainable': 85952, 'epoch': '2.414'} |
|
49%|███████████████████████████████████████████████▋ | 70/144 [01:16<01:16, 1.03s/it]
49%|████████████████████████████████████████████████▎ | 71/144 [01:17<01:14, 1.02s/it]
{'loss': '0.1159', 'grad_norm': '0.5551', 'learning_rate': '0.000109', 'ppl': '1.123', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '816.8', 'tokens/total': 579520, 'tokens/trainable': 87204, 'epoch': '2.448'} |
|
49%|████████████████████████████████████████████████▎ | 71/144 [01:17<01:14, 1.02s/it]
50%|█████████████████████████████████████████████████ | 72/144 [01:18<01:11, 1.00it/s]
{'loss': '0.08936', 'grad_norm': '0.2856', 'learning_rate': '0.0001067', 'ppl': '1.093', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '575.2', 'tokens/total': 586944, 'tokens/trainable': 88106, 'epoch': '2.483'} |
|
50%|█████████████████████████████████████████████████ | 72/144 [01:18<01:11, 1.00it/s]
51%|█████████████████████████████████████████████████▋ | 73/144 [01:19<01:09, 1.03it/s]
{'loss': '0.09291', 'grad_norm': '0.365', 'learning_rate': '0.0001045', 'ppl': '1.097', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '776.9', 'tokens/total': 594112, 'tokens/trainable': 89224, 'epoch': '2.517'} |
|
51%|█████████████████████████████████████████████████▋ | 73/144 [01:19<01:09, 1.03it/s]
51%|██████████████████████████████████████████████████▎ | 74/144 [01:20<01:11, 1.01s/it]
{'loss': '0.08388', 'grad_norm': '0.3015', 'learning_rate': '0.0001022', 'ppl': '1.087', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '1145', 'tokens/total': 602816, 'tokens/trainable': 91006, 'epoch': '2.552'} |
|
51%|██████████████████████████████████████████████████▎ | 74/144 [01:20<01:11, 1.01s/it]
52%|███████████████████████████████████████████████████ | 75/144 [01:21<01:10, 1.02s/it]
{'loss': '0.09955', 'grad_norm': '0.5344', 'learning_rate': '0.0001', 'ppl': '1.105', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '474.3', 'tokens/total': 611008, 'tokens/trainable': 92185, 'epoch': '2.586'} |
|
52%|███████████████████████████████████████████████████ | 75/144 [01:21<01:10, 1.02s/it]
53%|███████████████████████████████████████████████████▋ | 76/144 [01:22<01:12, 1.07s/it]
{'loss': '0.0959', 'grad_norm': '0.3283', 'learning_rate': '9.776e-05', 'ppl': '1.101', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '578.4', 'tokens/total': 620224, 'tokens/trainable': 93907, 'epoch': '2.621'} |
|
53%|███████████████████████████████████████████████████▋ | 76/144 [01:22<01:12, 1.07s/it]
53%|████████████████████████████████████████████████████▍ | 77/144 [01:23<01:12, 1.08s/it]
{'loss': '0.08367', 'grad_norm': '0.3295', 'learning_rate': '9.551e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '599.6', 'tokens/total': 628928, 'tokens/trainable': 95437, 'epoch': '2.655'} |
|
53%|████████████████████████████████████████████████████▍ | 77/144 [01:23<01:12, 1.08s/it]
54%|█████████████████████████████████████████████████████ | 78/144 [01:24<01:12, 1.10s/it]
{'loss': '0.0872', 'grad_norm': '0.3164', 'learning_rate': '9.327e-05', 'ppl': '1.091', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '446.9', 'tokens/total': 637888, 'tokens/trainable': 96555, 'epoch': '2.69'} |
|
54%|█████████████████████████████████████████████████████ | 78/144 [01:24<01:12, 1.10s/it]
55%|█████████████████████████████████████████████████████▊ | 79/144 [01:25<01:08, 1.05s/it]
{'loss': '0.09911', 'grad_norm': '0.3014', 'learning_rate': '9.104e-05', 'ppl': '1.104', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '674.6', 'tokens/total': 645312, 'tokens/trainable': 97790, 'epoch': '2.724'} |
|
55%|█████████████████████████████████████████████████████▊ | 79/144 [01:25<01:08, 1.05s/it]
56%|██████████████████████████████████████████████████████▍ | 80/144 [01:26<01:04, 1.01s/it]
{'loss': '0.115', 'grad_norm': '0.3922', 'learning_rate': '8.88e-05', 'ppl': '1.122', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '422.2', 'tokens/total': 652736, 'tokens/trainable': 98772, 'epoch': '2.759'} |
|
56%|██████████████████████████████████████████████████████▍ | 80/144 [01:26<01:04, 1.01s/it]
56%|███████████████████████████████████████████████████████▏ | 81/144 [01:27<01:06, 1.05s/it]
{'loss': '0.08908', 'grad_norm': '0.3826', 'learning_rate': '8.658e-05', 'ppl': '1.093', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '605.6', 'tokens/total': 661952, 'tokens/trainable': 100149, 'epoch': '2.793'} |
|
56%|███████████████████████████████████████████████████████▏ | 81/144 [01:27<01:06, 1.05s/it]
57%|███████████████████████████████████████████████████████▊ | 82/144 [01:28<01:07, 1.09s/it]
{'loss': '0.08324', 'grad_norm': '0.2812', 'learning_rate': '8.436e-05', 'ppl': '1.087', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '589.5', 'tokens/total': 671168, 'tokens/trainable': 101656, 'epoch': '2.828'} |
|
57%|███████████████████████████████████████████████████████▊ | 82/144 [01:28<01:07, 1.09s/it]
58%|████████████████████████████████████████████████████████▍ | 83/144 [01:29<01:05, 1.07s/it]
{'loss': '0.1029', 'grad_norm': '0.3454', 'learning_rate': '8.214e-05', 'ppl': '1.108', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '910.7', 'tokens/total': 679360, 'tokens/trainable': 103158, 'epoch': '2.862'} |
|
58%|████████████████████████████████████████████████████████▍ | 83/144 [01:29<01:05, 1.07s/it]
58%|█████████████████████████████████████████████████████████▏ | 84/144 [01:30<01:06, 1.11s/it]
{'loss': '0.09776', 'grad_norm': '0.3838', 'learning_rate': '7.994e-05', 'ppl': '1.103', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '563.4', 'tokens/total': 688832, 'tokens/trainable': 104296, 'epoch': '2.897'} |
|
58%|█████████████████████████████████████████████████████████▏ | 84/144 [01:30<01:06, 1.11s/it]
59%|█████████████████████████████████████████████████████████▊ | 85/144 [01:31<01:00, 1.02s/it]
{'loss': '0.09194', 'grad_norm': '0.3443', 'learning_rate': '7.775e-05', 'ppl': '1.096', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '842.7', 'tokens/total': 695488, 'tokens/trainable': 105395, 'epoch': '2.931'} |
|
59%|█████████████████████████████████████████████████████████▊ | 85/144 [01:31<01:00, 1.02s/it]
60%|██████████████████████████████████████████████████████████▌ | 86/144 [01:32<00:56, 1.02it/s]
{'loss': '0.06865', 'grad_norm': '0.3011', 'learning_rate': '7.557e-05', 'ppl': '1.071', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '333.2', 'tokens/total': 702656, 'tokens/trainable': 106404, 'epoch': '2.966'} |
|
60%|██████████████████████████████████████████████████████████▌ | 86/144 [01:32<00:56, 1.02it/s]
60%|███████████████████████████████████████████████████████████▏ | 87/144 [01:33<00:49, 1.14it/s]
{'loss': '0.09969', 'grad_norm': '0.5199', 'learning_rate': '7.34e-05', 'ppl': '1.105', 'memory/max_active (GiB)': '40.52', 'memory/max_allocated (GiB)': '40.52', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '160.2', 'tokens/total': 706304, 'tokens/trainable': 106800, 'epoch': '3'} |
|
60%|███████████████████████████████████████████████████████████▏ | 87/144 [01:33<00:49, 1.14it/s][2026-04-07 10:02:52,144] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-87 |
|
61%|███████████████████████████████████████████████████████████▉ | 88/144 [01:35<01:09, 1.25s/it]
{'loss': '0.06207', 'grad_norm': '0.2969', 'learning_rate': '7.124e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '462.3', 'tokens/total': 714240, 'tokens/trainable': 107984, 'epoch': '3.034'} |
|
61%|███████████████████████████████████████████████████████████▉ | 88/144 [01:35<01:09, 1.25s/it]
62%|████████████████████████████████████████████████████████████▌ | 89/144 [01:36<01:05, 1.19s/it]
{'loss': '0.04656', 'grad_norm': '0.1968', 'learning_rate': '6.91e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '430.5', 'tokens/total': 722688, 'tokens/trainable': 109321, 'epoch': '3.069'} |
|
62%|████████████████████████████████████████████████████████████▌ | 89/144 [01:36<01:05, 1.19s/it]
62%|█████████████████████████████████████████████████████████████▎ | 90/144 [01:37<00:59, 1.11s/it]
{'loss': '0.05607', 'grad_norm': '0.1708', 'learning_rate': '6.697e-05', 'ppl': '1.058', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '544.7', 'tokens/total': 729856, 'tokens/trainable': 110426, 'epoch': '3.103'} |
|
62%|█████████████████████████████████████████████████████████████▎ | 90/144 [01:37<00:59, 1.11s/it]
63%|█████████████████████████████████████████████████████████████▉ | 91/144 [01:38<00:55, 1.05s/it]
{'loss': '0.06276', 'grad_norm': '0.2478', 'learning_rate': '6.486e-05', 'ppl': '1.065', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '421.2', 'tokens/total': 737024, 'tokens/trainable': 111355, 'epoch': '3.138'} |
|
63%|█████████████████████████████████████████████████████████████▉ | 91/144 [01:38<00:55, 1.05s/it]
64%|██████████████████████████████████████████████████████████████▌ | 92/144 [01:39<00:52, 1.01s/it]
{'loss': '0.06717', 'grad_norm': '0.2365', 'learning_rate': '6.277e-05', 'ppl': '1.069', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '403.1', 'tokens/total': 744448, 'tokens/trainable': 112557, 'epoch': '3.172'} |
|
64%|██████████████████████████████████████████████████████████████▌ | 92/144 [01:39<00:52, 1.01s/it]
65%|███████████████████████████████████████████████████████████████▎ | 93/144 [01:40<00:52, 1.02s/it]
{'loss': '0.06895', 'grad_norm': '0.2605', 'learning_rate': '6.07e-05', 'ppl': '1.071', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '704.4', 'tokens/total': 752640, 'tokens/trainable': 113772, 'epoch': '3.207'} |
|
65%|███████████████████████████████████████████████████████████████▎ | 93/144 [01:40<00:52, 1.02s/it]
65%|███████████████████████████████████████████████████████████████▉ | 94/144 [01:41<00:54, 1.09s/it]
{'loss': '0.06202', 'grad_norm': '0.2754', 'learning_rate': '5.864e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '337.1', 'tokens/total': 762368, 'tokens/trainable': 114848, 'epoch': '3.241'} |
|
65%|███████████████████████████████████████████████████████████████▉ | 94/144 [01:41<00:54, 1.09s/it]
66%|████████████████████████████████████████████████████████████████▋ | 95/144 [01:42<00:50, 1.03s/it]
{'loss': '0.04728', 'grad_norm': '0.3247', 'learning_rate': '5.661e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '448.3', 'tokens/total': 769536, 'tokens/trainable': 115669, 'epoch': '3.276'} |
|
66%|████████████████████████████████████████████████████████████████▋ | 95/144 [01:42<00:50, 1.03s/it]
67%|█████████████████████████████████████████████████████████████████▎ | 96/144 [01:43<00:48, 1.02s/it]
{'loss': '0.07257', 'grad_norm': '0.3041', 'learning_rate': '5.46e-05', 'ppl': '1.075', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '554.1', 'tokens/total': 777216, 'tokens/trainable': 116758, 'epoch': '3.31'} |
|
67%|█████████████████████████████████████████████████████████████████▎ | 96/144 [01:43<00:48, 1.02s/it]
67%|██████████████████████████████████████████████████████████████████ | 97/144 [01:44<00:48, 1.03s/it]
{'loss': '0.0398', 'grad_norm': '0.2143', 'learning_rate': '5.261e-05', 'ppl': '1.041', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '450', 'tokens/total': 785664, 'tokens/trainable': 118167, 'epoch': '3.345'} |
|
67%|██████████████████████████████████████████████████████████████████ | 97/144 [01:44<00:48, 1.03s/it]
68%|██████████████████████████████████████████████████████████████████▋ | 98/144 [01:45<00:49, 1.07s/it]
{'loss': '0.06225', 'grad_norm': '0.2198', 'learning_rate': '5.065e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '60.88', 'memory/max_allocated (GiB)': '60.88', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '740.2', 'tokens/total': 794880, 'tokens/trainable': 119894, 'epoch': '3.379'} |
|
68%|██████████████████████████████████████████████████████████████████▋ | 98/144 [01:45<00:49, 1.07s/it]
69%|███████████████████████████████████████████████████████████████████▍ | 99/144 [01:46<00:48, 1.09s/it]
{'loss': '0.06667', 'grad_norm': '0.3708', 'learning_rate': '4.871e-05', 'ppl': '1.069', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '1179', 'tokens/total': 804096, 'tokens/trainable': 121868, 'epoch': '3.414'} |
|
69%|███████████████████████████████████████████████████████████████████▍ | 99/144 [01:46<00:48, 1.09s/it]
69%|███████████████████████████████████████████████████████████████████▎ | 100/144 [01:48<00:50, 1.14s/it]
{'loss': '0.05494', 'grad_norm': '0.2345', 'learning_rate': '4.68e-05', 'ppl': '1.056', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '697.1', 'tokens/total': 814336, 'tokens/trainable': 123772, 'epoch': '3.448'} |
|
69%|███████████████████████████████████████████████████████████████████▎ | 100/144 [01:48<00:50, 1.14s/it]
70%|████████████████████████████████████████████████████████████████████ | 101/144 [01:49<00:48, 1.12s/it]
{'loss': '0.07039', 'grad_norm': '0.2905', 'learning_rate': '4.491e-05', 'ppl': '1.073', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '342.7', 'tokens/total': 822784, 'tokens/trainable': 124818, 'epoch': '3.483'} |
|
70%|████████████████████████████████████████████████████████████████████ | 101/144 [01:49<00:48, 1.12s/it]
71%|████████████████████████████████████████████████████████████████████▋ | 102/144 [01:50<00:46, 1.10s/it]
{'loss': '0.0404', 'grad_norm': '0.2652', 'learning_rate': '4.305e-05', 'ppl': '1.041', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '587.8', 'tokens/total': 831232, 'tokens/trainable': 125777, 'epoch': '3.517'} |
|
71%|████████████████████████████████████████████████████████████████████▋ | 102/144 [01:50<00:46, 1.10s/it]
72%|█████████████████████████████████████████████████████████████████████▍ | 103/144 [01:51<00:42, 1.05s/it]
{'loss': '0.04841', 'grad_norm': '0.3569', 'learning_rate': '4.122e-05', 'ppl': '1.05', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '476.6', 'tokens/total': 838656, 'tokens/trainable': 126543, 'epoch': '3.552'} |
|
72%|█████████████████████████████████████████████████████████████████████▍ | 103/144 [01:51<00:42, 1.05s/it]
72%|██████████████████████████████████████████████████████████████████████ | 104/144 [01:52<00:41, 1.03s/it]
{'loss': '0.06206', 'grad_norm': '0.3093', 'learning_rate': '3.942e-05', 'ppl': '1.064', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '572.6', 'tokens/total': 846592, 'tokens/trainable': 127626, 'epoch': '3.586'} |
|
72%|██████████████████████████████████████████████████████████████████████ | 104/144 [01:52<00:41, 1.03s/it]
73%|██████████████████████████████████████████████████████████████████████▋ | 105/144 [01:53<00:40, 1.03s/it]
{'loss': '0.05195', 'grad_norm': '0.2954', 'learning_rate': '3.765e-05', 'ppl': '1.053', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '587.7', 'tokens/total': 854528, 'tokens/trainable': 128875, 'epoch': '3.621'} |
|
73%|██████████████████████████████████████████████████████████████████████▋ | 105/144 [01:53<00:40, 1.03s/it]
74%|███████████████████████████████████████████████████████████████████████▍ | 106/144 [01:54<00:38, 1.02s/it]
{'loss': '0.0636', 'grad_norm': '0.4605', 'learning_rate': '3.591e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '52.68', 'memory/max_allocated (GiB)': '52.68', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '845.8', 'tokens/total': 862464, 'tokens/trainable': 130189, 'epoch': '3.655'} |
|
74%|███████████████████████████████████████████████████████████████████████▍ | 106/144 [01:54<00:38, 1.02s/it]
74%|████████████████████████████████████████████████████████████████████████ | 107/144 [01:55<00:37, 1.01s/it]
{'loss': '0.05174', 'grad_norm': '0.3131', 'learning_rate': '3.421e-05', 'ppl': '1.053', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '874.8', 'tokens/total': 870400, 'tokens/trainable': 131508, 'epoch': '3.69'} |
|
74%|████████████████████████████████████████████████████████████████████████ | 107/144 [01:55<00:37, 1.01s/it]
75%|████████████████████████████████████████████████████████████████████████▊ | 108/144 [01:56<00:38, 1.06s/it]
{'loss': '0.03808', 'grad_norm': '0.2428', 'learning_rate': '3.253e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '691.1', 'tokens/total': 879616, 'tokens/trainable': 132891, 'epoch': '3.724'} |
|
75%|████████████████████████████████████████████████████████████████████████▊ | 108/144 [01:56<00:38, 1.06s/it]
76%|█████████████████████████████████████████████████████████████████████████▍ | 109/144 [01:57<00:39, 1.12s/it]
{'loss': '0.06047', 'grad_norm': '0.2851', 'learning_rate': '3.089e-05', 'ppl': '1.062', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '623.6', 'tokens/total': 889600, 'tokens/trainable': 134137, 'epoch': '3.759'} |
|
76%|█████████████████████████████████████████████████████████████████████████▍ | 109/144 [01:57<00:39, 1.12s/it]
76%|██████████████████████████████████████████████████████████████████████████ | 110/144 [01:58<00:36, 1.06s/it]
{'loss': '0.06354', 'grad_norm': '0.2822', 'learning_rate': '2.929e-05', 'ppl': '1.066', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '577.8', 'tokens/total': 897024, 'tokens/trainable': 135094, 'epoch': '3.793'} |
|
76%|██████████████████████████████████████████████████████████████████████████ | 110/144 [01:58<00:36, 1.06s/it]
77%|██████████████████████████████████████████████████████████████████████████▊ | 111/144 [01:59<00:35, 1.06s/it]
{'loss': '0.03931', 'grad_norm': '0.2651', 'learning_rate': '2.772e-05', 'ppl': '1.04', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '794', 'tokens/total': 905472, 'tokens/trainable': 136713, 'epoch': '3.828'} |
|
77%|██████████████████████████████████████████████████████████████████████████▊ | 111/144 [01:59<00:35, 1.06s/it]
78%|███████████████████████████████████████████████████████████████████████████▍ | 112/144 [02:00<00:33, 1.04s/it]
{'loss': '0.0481', 'grad_norm': '0.3099', 'learning_rate': '2.619e-05', 'ppl': '1.049', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '568.8', 'tokens/total': 913152, 'tokens/trainable': 137675, 'epoch': '3.862'} |
|
78%|███████████████████████████████████████████████████████████████████████████▍ | 112/144 [02:00<00:33, 1.04s/it]
78%|████████████████████████████████████████████████████████████████████████████ | 113/144 [02:01<00:32, 1.06s/it]
{'loss': '0.0465', 'grad_norm': '0.633', 'learning_rate': '2.469e-05', 'ppl': '1.048', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '933.4', 'tokens/total': 922112, 'tokens/trainable': 139199, 'epoch': '3.897'} |
|
78%|████████████████████████████████████████████████████████████████████████████ | 113/144 [02:01<00:32, 1.06s/it]
79%|████████████████████████████████████████████████████████████████████████████▊ | 114/144 [02:02<00:31, 1.03s/it]
{'loss': '0.05563', 'grad_norm': '0.2643', 'learning_rate': '2.324e-05', 'ppl': '1.057', 'memory/max_active (GiB)': '46.81', 'memory/max_allocated (GiB)': '46.81', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '648.3', 'tokens/total': 929792, 'tokens/trainable': 140480, 'epoch': '3.931'} |
|
79%|████████████████████████████████████████████████████████████████████████████▊ | 114/144 [02:02<00:31, 1.03s/it]
80%|█████████████████████████████████████████████████████████████████████████████▍ | 115/144 [02:03<00:28, 1.01it/s]
{'loss': '0.05304', 'grad_norm': '0.3273', 'learning_rate': '2.182e-05', 'ppl': '1.054', 'memory/max_active (GiB)': '44.76', 'memory/max_allocated (GiB)': '44.76', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '417.2', 'tokens/total': 936704, 'tokens/trainable': 141549, 'epoch': '3.966'} |
|
80%|█████████████████████████████████████████████████████████████████████████████▍ | 115/144 [02:03<00:28, 1.01it/s]
81%|██████████████████████████████████████████████████████████████████████████████▏ | 116/144 [02:04<00:25, 1.09it/s]
{'loss': '0.0386', 'grad_norm': '0.396', 'learning_rate': '2.044e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '383.7', 'tokens/total': 941760, 'tokens/trainable': 142400, 'epoch': '4'} |
|
81%|██████████████████████████████████████████████████████████████████████████████▏ | 116/144 [02:04<00:25, 1.09it/s][2026-04-07 10:03:23,028] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-116 |
|
81%|██████████████████████████████████████████████████████████████████████████████▊ | 117/144 [02:06<00:34, 1.26s/it]
{'loss': '0.02968', 'grad_norm': '0.1892', 'learning_rate': '1.91e-05', 'ppl': '1.03', 'memory/max_active (GiB)': '48.41', 'memory/max_allocated (GiB)': '48.41', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '352.6', 'tokens/total': 949696, 'tokens/trainable': 143669, 'epoch': '4.034'} |
|
81%|██████████████████████████████████████████████████████████████████████████████▊ | 117/144 [02:06<00:34, 1.26s/it]
82%|███████████████████████████████████████████████████████████████████████████████▍ | 118/144 [02:07<00:31, 1.22s/it]
{'loss': '0.03344', 'grad_norm': '0.2217', 'learning_rate': '1.78e-05', 'ppl': '1.034', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '680.2', 'tokens/total': 958912, 'tokens/trainable': 145086, 'epoch': '4.069'} |
|
82%|███████████████████████████████████████████████████████████████████████████████▍ | 118/144 [02:07<00:31, 1.22s/it]
83%|████████████████████████████████████████████████████████████████████████████████▏ | 119/144 [02:08<00:29, 1.18s/it]
{'loss': '0.03798', 'grad_norm': '0.2478', 'learning_rate': '1.654e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '397.6', 'tokens/total': 967616, 'tokens/trainable': 145921, 'epoch': '4.103'} |
|
83%|████████████████████████████████████████████████████████████████████████████████▏ | 119/144 [02:08<00:29, 1.18s/it]
83%|████████████████████████████████████████████████████████████████████████████████▊ | 120/144 [02:09<00:27, 1.16s/it]
{'loss': '0.03804', 'grad_norm': '0.5864', 'learning_rate': '1.533e-05', 'ppl': '1.039', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '294.9', 'tokens/total': 976320, 'tokens/trainable': 146998, 'epoch': '4.138'} |
|
83%|████████████████████████████████████████████████████████████████████████████████▊ | 120/144 [02:09<00:27, 1.16s/it]
84%|█████████████████████████████████████████████████████████████████████████████████▌ | 121/144 [02:10<00:27, 1.18s/it]
{'loss': '0.02537', 'grad_norm': '0.2323', 'learning_rate': '1.416e-05', 'ppl': '1.026', 'memory/max_active (GiB)': '60.72', 'memory/max_allocated (GiB)': '60.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '704.1', 'tokens/total': 986048, 'tokens/trainable': 148666, 'epoch': '4.172'} |
|
84%|█████████████████████████████████████████████████████████████████████████████████▌ | 121/144 [02:10<00:27, 1.18s/it]
85%|██████████████████████████████████████████████████████████████████████████████████▏ | 122/144 [02:11<00:23, 1.08s/it]
{'loss': '0.03578', 'grad_norm': '0.2683', 'learning_rate': '1.303e-05', 'ppl': '1.036', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '662.6', 'tokens/total': 992704, 'tokens/trainable': 149800, 'epoch': '4.207'} |
|
85%|██████████████████████████████████████████████████████████████████████████████████▏ | 122/144 [02:11<00:23, 1.08s/it]
85%|██████████████████████████████████████████████████████████████████████████████████▊ | 123/144 [02:12<00:22, 1.05s/it]
{'loss': '0.03225', 'grad_norm': '0.4071', 'learning_rate': '1.194e-05', 'ppl': '1.033', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '571.1', 'tokens/total': 1000640, 'tokens/trainable': 150963, 'epoch': '4.241'} |
|
85%|██████████████████████████████████████████████████████████████████████████████████▊ | 123/144 [02:12<00:22, 1.05s/it]
86%|███████████████████████████████████████████████████████████████████████████████████▌ | 124/144 [02:13<00:21, 1.06s/it]
{'loss': '0.02882', 'grad_norm': '0.1641', 'learning_rate': '1.09e-05', 'ppl': '1.029', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '750.5', 'tokens/total': 1009088, 'tokens/trainable': 152496, 'epoch': '4.276'} |
|
86%|███████████████████████████████████████████████████████████████████████████████████▌ | 124/144 [02:13<00:21, 1.06s/it]
87%|████████████████████████████████████████████████████████████████████████████████████▏ | 125/144 [02:14<00:21, 1.12s/it]
{'loss': '0.03131', 'grad_norm': '0.1728', 'learning_rate': '9.903e-06', 'ppl': '1.032', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '705.2', 'tokens/total': 1019072, 'tokens/trainable': 153857, 'epoch': '4.31'} |
|
87%|████████████████████████████████████████████████████████████████████████████████████▏ | 125/144 [02:14<00:21, 1.12s/it]
88%|████████████████████████████████████████████████████████████████████████████████████▉ | 126/144 [02:16<00:20, 1.11s/it]
{'loss': '0.02575', 'grad_norm': '0.1367', 'learning_rate': '8.952e-06', 'ppl': '1.026', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '772.3', 'tokens/total': 1027776, 'tokens/trainable': 155521, 'epoch': '4.345'} |
|
88%|████████████████████████████████████████████████████████████████████████████████████▉ | 126/144 [02:16<00:20, 1.11s/it]
88%|█████████████████████████████████████████████████████████████████████████████████████▌ | 127/144 [02:17<00:18, 1.10s/it]
{'loss': '0.03918', 'grad_norm': '0.2347', 'learning_rate': '8.047e-06', 'ppl': '1.04', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '516.6', 'tokens/total': 1036224, 'tokens/trainable': 156435, 'epoch': '4.379'} |
|
88%|█████████████████████████████████████████████████████████████████████████████████████▌ | 127/144 [02:17<00:18, 1.10s/it]
89%|██████████████████████████████████████████████████████████████████████████████████████▏ | 128/144 [02:18<00:17, 1.11s/it]
{'loss': '0.0305', 'grad_norm': '0.2856', 'learning_rate': '7.189e-06', 'ppl': '1.031', 'memory/max_active (GiB)': '54.56', 'memory/max_allocated (GiB)': '54.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '321.6', 'tokens/total': 1045184, 'tokens/trainable': 157278, 'epoch': '4.414'} |
|
89%|██████████████████████████████████████████████████████████████████████████████████████▏ | 128/144 [02:18<00:17, 1.11s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████▉ | 129/144 [02:19<00:16, 1.12s/it]
{'loss': '0.02478', 'grad_norm': '0.1711', 'learning_rate': '6.377e-06', 'ppl': '1.025', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '659', 'tokens/total': 1054400, 'tokens/trainable': 158505, 'epoch': '4.448'} |
|
90%|██████████████████████████████████████████████████████████████████████████████████████▉ | 129/144 [02:19<00:16, 1.12s/it]
90%|███████████████████████████████████████████████████████████████████████████████████████▌ | 130/144 [02:20<00:15, 1.11s/it]
{'loss': '0.04704', 'grad_norm': '0.2165', 'learning_rate': '5.612e-06', 'ppl': '1.048', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '617.7', 'tokens/total': 1063104, 'tokens/trainable': 159759, 'epoch': '4.483'} |
|
90%|███████████████████████████████████████████████████████████████████████████████████████▌ | 130/144 [02:20<00:15, 1.11s/it]
91%|████████████████████████████████████████████████████████████████████████████████████████▏ | 131/144 [02:21<00:14, 1.09s/it]
{'loss': '0.02228', 'grad_norm': '0.1575', 'learning_rate': '4.894e-06', 'ppl': '1.023', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '587.2', 'tokens/total': 1071296, 'tokens/trainable': 161371, 'epoch': '4.517'} |
|
91%|████████████████████████████████████████████████████████████████████████████████████████▏ | 131/144 [02:21<00:14, 1.09s/it]
92%|████████████████████████████████████████████████████████████████████████████████████████▉ | 132/144 [02:22<00:13, 1.08s/it]
{'loss': '0.03479', 'grad_norm': '0.1925', 'learning_rate': '4.225e-06', 'ppl': '1.035', 'memory/max_active (GiB)': '54.72', 'memory/max_allocated (GiB)': '54.72', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '793.1', 'tokens/total': 1079744, 'tokens/trainable': 162649, 'epoch': '4.552'} |
|
92%|████████████████████████████████████████████████████████████████████████████████████████▉ | 132/144 [02:22<00:13, 1.08s/it]
92%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 133/144 [02:23<00:11, 1.05s/it]
{'loss': '0.02817', 'grad_norm': '0.1716', 'learning_rate': '3.604e-06', 'ppl': '1.029', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '590.8', 'tokens/total': 1087424, 'tokens/trainable': 163754, 'epoch': '4.586'} |
|
92%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 133/144 [02:23<00:11, 1.05s/it]
93%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 134/144 [02:24<00:10, 1.05s/it]
{'loss': '0.03311', 'grad_norm': '0.2505', 'learning_rate': '3.031e-06', 'ppl': '1.034', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '453.8', 'tokens/total': 1095616, 'tokens/trainable': 165030, 'epoch': '4.621'} |
|
93%|██████████████████████████████████████████████████████████████████████████████████████████▎ | 134/144 [02:24<00:10, 1.05s/it]
94%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 135/144 [02:25<00:09, 1.05s/it]
{'loss': '0.03378', 'grad_norm': '0.2401', 'learning_rate': '2.507e-06', 'ppl': '1.034', 'memory/max_active (GiB)': '50.47', 'memory/max_allocated (GiB)': '50.47', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '618.7', 'tokens/total': 1103808, 'tokens/trainable': 166465, 'epoch': '4.655'} |
|
94%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 135/144 [02:25<00:09, 1.05s/it]
94%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 136/144 [02:26<00:08, 1.06s/it]
{'loss': '0.03617', 'grad_norm': '0.2443', 'learning_rate': '2.032e-06', 'ppl': '1.037', 'memory/max_active (GiB)': '56.77', 'memory/max_allocated (GiB)': '56.77', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '573.1', 'tokens/total': 1112512, 'tokens/trainable': 167478, 'epoch': '4.69'} |
|
94%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 136/144 [02:26<00:08, 1.06s/it]
95%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 137/144 [02:27<00:07, 1.04s/it]
{'loss': '0.03891', 'grad_norm': '0.2393', 'learning_rate': '1.607e-06', 'ppl': '1.04', 'memory/max_active (GiB)': '50.64', 'memory/max_allocated (GiB)': '50.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '841.8', 'tokens/total': 1120448, 'tokens/trainable': 168824, 'epoch': '4.724'} |
|
95%|████████████████████████████████████████████████████████████████████████████████████████████▎ | 137/144 [02:27<00:07, 1.04s/it]
96%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 138/144 [02:28<00:06, 1.07s/it]
{'loss': '0.02679', 'grad_norm': '0.2252', 'learning_rate': '1.231e-06', 'ppl': '1.027', 'memory/max_active (GiB)': '52.51', 'memory/max_allocated (GiB)': '52.51', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '718.7', 'tokens/total': 1129408, 'tokens/trainable': 170354, 'epoch': '4.759'} |
|
96%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 138/144 [02:28<00:06, 1.07s/it]
97%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 139/144 [02:29<00:04, 1.01it/s]
{'loss': '0.03431', 'grad_norm': '0.2079', 'learning_rate': '9.05e-07', 'ppl': '1.035', 'memory/max_active (GiB)': '42.56', 'memory/max_allocated (GiB)': '42.56', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '658.7', 'tokens/total': 1135808, 'tokens/trainable': 171513, 'epoch': '4.793'} |
|
97%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 139/144 [02:29<00:04, 1.01it/s]
97%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 140/144 [02:30<00:04, 1.02s/it]
{'loss': '0.02909', 'grad_norm': '0.1657', 'learning_rate': '6.288e-07', 'ppl': '1.03', 'memory/max_active (GiB)': '56.6', 'memory/max_allocated (GiB)': '56.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '753', 'tokens/total': 1144512, 'tokens/trainable': 172991, 'epoch': '4.828'} |
|
97%|██████████████████████████████████████████████████████████████████████████████████████████████▎ | 140/144 [02:30<00:04, 1.02s/it]
98%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 141/144 [02:31<00:02, 1.01it/s]
{'loss': '0.02993', 'grad_norm': '0.1878', 'learning_rate': '4.026e-07', 'ppl': '1.03', 'memory/max_active (GiB)': '46.64', 'memory/max_allocated (GiB)': '46.64', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '438', 'tokens/total': 1151936, 'tokens/trainable': 173991, 'epoch': '4.862'} |
|
98%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 141/144 [02:31<00:02, 1.01it/s]
99%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 142/144 [02:32<00:01, 1.04it/s]
{'loss': '0.03719', 'grad_norm': '0.3583', 'learning_rate': '2.265e-07', 'ppl': '1.038', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '742', 'tokens/total': 1159104, 'tokens/trainable': 174948, 'epoch': '4.897'} |
|
99%|███████████████████████████████████████████████████████████████████████████████████████████████▋ | 142/144 [02:32<00:01, 1.04it/s]
99%|████████████████████████████████████████████████████████████████████████████████████████████████▎| 143/144 [02:33<00:00, 1.05it/s]
{'loss': '0.02699', 'grad_norm': '0.144', 'learning_rate': '1.007e-07', 'ppl': '1.027', 'memory/max_active (GiB)': '48.58', 'memory/max_allocated (GiB)': '48.58', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '682.9', 'tokens/total': 1166528, 'tokens/trainable': 176344, 'epoch': '4.931'} |
|
99%|████████████████████████████████████████████████████████████████████████████████████████████████▎| 143/144 [02:33<00:00, 1.05it/s]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:34<00:00, 1.10it/s]
{'loss': '0.03761', 'grad_norm': '0.1827', 'learning_rate': '2.518e-08', 'ppl': '1.038', 'memory/max_active (GiB)': '44.6', 'memory/max_allocated (GiB)': '44.6', 'memory/device_reserved (GiB)': '62.72', 'tokens/train_per_sec_per_gpu': '470.5', 'tokens/total': 1172928, 'tokens/trainable': 177353, 'epoch': '4.966'} |
|
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:34<00:00, 1.10it/s][2026-04-07 10:03:53,154] [INFO] [axolotl.core.trainers.base._save:776] [PID:10658] Saving model checkpoint to /workspace/outputs/ministral8b-tsql/checkpoint-144 |
|
{'train_runtime': '155.1', 'train_samples_per_second': '7.427', 'train_steps_per_second': '0.928', 'train_loss': '0.1373', 'memory/max_active (GiB)': '16.96', 'memory/max_allocated (GiB)': '16.96', 'memory/device_reserved (GiB)': '62.72', 'epoch': '4.966', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:35<00:00, 1.10it/s]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 144/144 [02:35<00:00, 1.08s/it] |
| [2026-04-07 10:03:54,163] [INFO] [axolotl.train.save_trained_model:241] [PID:10658] Training completed! Saving trained model to /workspace/outputs/ministral8b-tsql. |
| [2026-04-07 10:03:54,446] [INFO] [axolotl.train.save_trained_model:355] [PID:10658] Model successfully saved to /workspace/outputs/ministral8b-tsql |
| |
