File size: 66,446 Bytes
90dc82e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 | [2026-04-10 13:58:29,722] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:7899] baseline 0.000GB ()
[2026-04-10 13:58:29,722] [INFO] [axolotl.cli.config.load_cfg:248] [PID:7899] config:
{
"activation_offloading": false,
"adapter": "lora",
"axolotl_config_path": "config.yaml",
"base_model": "unsloth/Meta-Llama-3.1-8B-Instruct",
"base_model_config": "unsloth/Meta-Llama-3.1-8B-Instruct",
"batch_size": 16,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_90",
"fp8": false,
"n_gpu": 1,
"n_node": 1
},
"context_parallel_size": 1,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_processes": 8,
"datasets": [
{
"chat_template": "llama3",
"field_messages": "messages",
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "mx003/cve_dataset_1000",
"trust_remote_code": false,
"type": "chat_template"
}
],
"ddp": false,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"env_capabilities": {
"torch_version": "2.7.1"
},
"eval_batch_size": 4,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_sample_packing": true,
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"flash_attention": true,
"fp16": false,
"gradient_accumulation_steps": 4,
"gradient_checkpointing": true,
"gradient_checkpointing_kwargs": {
"use_reentrant": true
},
"group_by_length": true,
"include_tkps": true,
"is_llama_derived_model": true,
"learning_rate": 0.0002,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": true,
"load_in_8bit": false,
"local_rank": 0,
"lora_alpha": 64,
"lora_dropout": 0.05,
"lora_modules_to_save": [
"embed_tokens",
"lm_head"
],
"lora_r": 32,
"lora_target_modules": [
"q_proj",
"v_proj",
"k_proj",
"o_proj",
"gate_proj",
"down_proj",
"up_proj"
],
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "cosine",
"mean_resizing_embeddings": false,
"micro_batch_size": 4,
"model_config_type": "llama",
"num_epochs": 2.0,
"optimizer": "adamw_torch_fused",
"output_dir": "./outputs/mymodel",
"pad_to_sequence_len": true,
"pretrain_multipack_attn": true,
"profiler_steps_start": 0,
"qlora_sharded_model_loading": false,
"ray_num_workers": 1,
"resources_per_worker": {
"GPU": 1
},
"sample_packing": true,
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": false,
"save_safetensors": true,
"save_steps": 50,
"sequence_len": 4096,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"special_tokens": {
"pad_token": "<|end_of_text|>"
},
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "unsloth/Meta-Llama-3.1-8B-Instruct",
"tokenizer_save_jinja_files": true,
"tokens": [
"<|end_of_text|>"
],
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"log_completions": false,
"mask_truncated_completions": false,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"scale_rewards": true,
"sync_ref_model": false,
"use_vllm": false,
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"use_ray": false,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"gpu_memory_utilization": 0.9,
"host": "0.0.0.0",
"port": 8000
},
"weight_decay": 0.0,
"world_size": 1
}
[2026-04-10 13:58:30,702] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:7899] EOS: 128009 / <|eot_id|>
[2026-04-10 13:58:30,702] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:7899] BOS: 128000 / <|begin_of_text|>
[2026-04-10 13:58:30,702] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:7899] PAD: 128001 / <|end_of_text|>
[2026-04-10 13:58:30,702] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:7899] UNK: None / None
[2026-04-10 13:58:30,703] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:476] [PID:7899] Unable to find prepared dataset in last_run_prepared/d1f1263ce25c23753621d9371f061ee3
[2026-04-10 13:58:30,703] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:7899] Loading raw datasets...
[2026-04-10 13:58:30,703] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:7899] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
[2026-04-10 13:58:33,488] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:7899] Loading dataset: mx003/cve_dataset_1000 with base_type: chat_template and prompt_style: None
[2026-04-10 13:58:33,490] [INFO] [axolotl.prompt_strategies.chat_template.__call__:969] [PID:7899] Using chat template:
---
{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>
' }}{% endif %}
---
Tokenizing Prompts (num_proc=8): 0%| | 0/989 [00:00<?, ? examples/s]
Tokenizing Prompts (num_proc=8): 13%|███████████▌ | 124/989 [00:01<00:12, 70.54 examples/s]
Tokenizing Prompts (num_proc=8): 25%|██████████████████████▊ | 248/989 [00:01<00:05, 145.06 examples/s]
Tokenizing Prompts (num_proc=8): 38%|██████████████████████████████████▏ | 372/989 [00:02<00:02, 236.49 examples/s]
Tokenizing Prompts (num_proc=8): 63%|█████████████████████████████████████████████████████████ | 620/989 [00:02<00:00, 482.05 examples/s]
Tokenizing Prompts (num_proc=8): 100%|███████████████████████████████████████████████████████████████████████████████████████████| 989/989 [00:02<00:00, 740.12 examples/s]
Tokenizing Prompts (num_proc=8): 100%|███████████████████████████████████████████████████████████████████████████████████████████| 989/989 [00:02<00:00, 378.73 examples/s]
[2026-04-10 13:58:36,220] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:218] [PID:7899] min_input_len: 489
[2026-04-10 13:58:36,220] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:220] [PID:7899] max_input_len: 22717
Dropping Long Sequences (>4096) (num_proc=8): 0%| | 0/989 [00:00<?, ? examples/s]
Dropping Long Sequences (>4096) (num_proc=8): 13%|█████████▊ | 124/989 [00:00<00:01, 719.26 examples/s]
Dropping Long Sequences (>4096) (num_proc=8): 100%|█████████████████████████████████████████████████████████████████████████████| 989/989 [00:00<00:00, 3388.59 examples/s]
[2026-04-10 13:58:36,528] [WARNING] [axolotl.utils.data.utils.handle_long_seq_in_dataset:260] [PID:7899] Dropped 55 samples from dataset
Drop Samples with Zero Trainable Tokens (num_proc=8): 0%| | 0/934 [00:00<?, ? examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=8): 13%|████████▊ | 117/934 [00:00<00:01, 724.01 examples/s]
Drop Samples with Zero Trainable Tokens (num_proc=8): 100%|█████████████████████████████████████████████████████████████████████| 934/934 [00:00<00:00, 3255.94 examples/s]
Group By Length (num_proc=8): 0%| | 0/934 [00:00<?, ? examples/s]
Group By Length (num_proc=8): 12%|███████████▌ | 115/934 [00:00<00:01, 717.61 examples/s]
Group By Length (num_proc=8): 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 934/934 [00:00<00:00, 2747.28 examples/s]
Add position_id column (Sample Packing) (num_proc=8): 0%| | 0/934 [00:00<?, ? examples/s]
Add position_id column (Sample Packing) (num_proc=8): 13%|████████▊ | 117/934 [00:00<00:01, 700.04 examples/s]
Add position_id column (Sample Packing) (num_proc=8): 100%|█████████████████████████████████████████████████████████████████████| 934/934 [00:00<00:00, 2998.02 examples/s]
Saving the dataset (0/3 shards): 0%| | 0/934 [00:00<?, ? examples/s]
Saving the dataset (1/3 shards): 67%|████████████████████████████████████████████████████████████ | 623/934 [00:00<00:00, 8737.55 examples/s]
Saving the dataset (2/3 shards): 67%|████████████████████████████████████████████████████████████ | 623/934 [00:00<00:00, 8624.87 examples/s]
Saving the dataset (3/3 shards): 100%|█████████████████████████████████████████████████████████████████████████████████████████| 934/934 [00:00<00:00, 12744.46 examples/s]
Saving the dataset (3/3 shards): 100%|██████████████████████████████████████████████████████████████████████████████████████████| 934/934 [00:00<00:00, 6729.26 examples/s]
[2026-04-10 13:58:37,675] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:404] [PID:7899] total_num_tokens: 1_457_482
[2026-04-10 13:58:37,684] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:422] [PID:7899] `total_supervised_tokens: 308_054`
[2026-04-10 13:58:38,798] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.5154547691345215
[2026-04-10 13:58:39,325] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.5266461372375488
[2026-04-10 13:58:39,857] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.53169846534729
[2026-04-10 13:58:40,395] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.5374758243560791
[2026-04-10 13:58:40,415] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:434] [PID:7899] gather_len_batches: [90]
[2026-04-10 13:58:40,415] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:481] [PID:7899] data_loader_len: 22
[2026-04-10 13:58:40,415] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:497] [PID:7899] sample_packing_eff_est across ranks: [0.9775565011160714]
[2026-04-10 13:58:40,415] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:509] [PID:7899] sample_packing_eff_est: 0.98
[2026-04-10 13:58:40,415] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:520] [PID:7899] total_num_steps: 44
[2026-04-10 13:58:40,415] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:7899] Maximum number of steps set at 44
[2026-04-10 13:58:40,445] [DEBUG] [axolotl.train.setup_model_and_tokenizer:65] [PID:7899] Loading tokenizer... unsloth/Meta-Llama-3.1-8B-Instruct
[2026-04-10 13:58:41,324] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:278] [PID:7899] EOS: 128009 / <|eot_id|>
[2026-04-10 13:58:41,324] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:279] [PID:7899] BOS: 128000 / <|begin_of_text|>
[2026-04-10 13:58:41,324] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:7899] PAD: 128001 / <|end_of_text|>
[2026-04-10 13:58:41,324] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:7899] UNK: None / None
[2026-04-10 13:58:41,324] [DEBUG] [axolotl.train.setup_model_and_tokenizer:74] [PID:7899] Loading model
[2026-04-10 13:58:41,491] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:7899] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-04-10 13:58:41,492] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:7899] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-04-10 13:58:41,492] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:301] [PID:7899] Applying multipack dataloader patch for sample packing...
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|███████████████████████████▎ | 1/4 [00:02<00:08, 2.94s/it]
Loading checkpoint shards: 50%|██████████████████████████████████████████████████████▌ | 2/4 [00:06<00:06, 3.29s/it]
Loading checkpoint shards: 75%|█████████████████████████████████████████████████████████████████████████████████▊ | 3/4 [00:09<00:03, 3.35s/it]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:10<00:00, 2.11s/it]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:10<00:00, 2.53s/it]
[2026-04-10 13:58:52,266] [INFO] [axolotl.loaders.model._prepare_model_for_quantization:863] [PID:7899] converting PEFT model w/ prepare_model_for_kbit_training
[2026-04-10 13:58:52,269] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:345] [PID:7899] Converting modules to torch.bfloat16
[2026-04-10 13:58:52,276] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:7899] Memory usage after model load 8.657GB (+8.657GB allocated, +9.924GB reserved)
trainable params: 1,134,559,232 || all params: 9,164,820,480 || trainable%: 12.3795
[2026-04-10 13:58:52,901] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:7899] after adapters 7.969GB (+7.969GB allocated, +10.072GB reserved)
[2026-04-10 13:58:57,835] [INFO] [axolotl.train.save_initial_configs:398] [PID:7899] Pre-saving adapter config to ./outputs/mymodel...
[2026-04-10 13:58:57,836] [INFO] [axolotl.train.save_initial_configs:402] [PID:7899] Pre-saving tokenizer to ./outputs/mymodel...
[2026-04-10 13:58:57,967] [INFO] [axolotl.train.save_initial_configs:407] [PID:7899] Pre-saving model config to ./outputs/mymodel...
[2026-04-10 13:58:57,969] [INFO] [axolotl.train.execute_training:196] [PID:7899] Starting trainer...
[2026-04-10 13:59:00,245] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.8013865947723389
[2026-04-10 13:59:01,035] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.7904410362243652
[2026-04-10 13:59:01,802] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.7667834758758545
[2026-04-10 13:59:02,580] [DEBUG] [axolotl.utils.samplers.multipack.__len__:458] [PID:7899] generate_batches time: 0.7771618366241455
[2026-04-10 13:59:02,580] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:434] [PID:7899] gather_len_batches: [90]
0%| | 0/44 [00:00<?, ?it/s]
2%|███ | 1/44 [00:12<09:17, 12.97s/it]
{'loss': 0.6377, 'grad_norm': 3.9988136291503906, 'learning_rate': 0.0, 'memory/max_active (GiB)': 38.73, 'memory/max_allocated (GiB)': 38.73, 'memory/device_reserved (GiB)': 46.38, 'tokens_per_second_per_gpu': 1272.18, 'epoch': 0.04}
2%|███ | 1/44 [00:12<09:17, 12.97s/it]
5%|██████▏ | 2/44 [00:23<08:04, 11.54s/it]
{'loss': 0.6022, 'grad_norm': 3.8201091289520264, 'learning_rate': 0.0001, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1310.05, 'epoch': 0.09}
5%|██████▏ | 2/44 [00:23<08:04, 11.54s/it]
7%|█████████▏ | 3/44 [00:34<07:34, 11.07s/it]
{'loss': 0.258, 'grad_norm': 2.0871596336364746, 'learning_rate': 0.0002, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1449.56, 'epoch': 0.13}
7%|█████████▏ | 3/44 [00:34<07:34, 11.07s/it]
9%|████████████▎ | 4/44 [00:44<07:14, 10.87s/it]
{'loss': 0.2098, 'grad_norm': 9.840972900390625, 'learning_rate': 0.00019972037971811802, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1242.92, 'epoch': 0.18}
9%|████████████▎ | 4/44 [00:44<07:14, 10.87s/it]
11%|███████████████▎ | 5/44 [00:55<06:59, 10.77s/it]
{'loss': 0.1372, 'grad_norm': 16.3769474029541, 'learning_rate': 0.00019888308262251285, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1221.02, 'epoch': 0.22}
11%|███████████████▎ | 5/44 [00:55<06:59, 10.77s/it]
14%|██████████████████▍ | 6/44 [01:05<06:46, 10.70s/it]
{'loss': 0.103, 'grad_norm': 1.9449656009674072, 'learning_rate': 0.00019749279121818235, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1230.89, 'epoch': 0.27}
14%|██████████████████▍ | 6/44 [01:05<06:46, 10.70s/it]
16%|█████████████████████▍ | 7/44 [01:16<06:33, 10.64s/it]
{'loss': 0.06, 'grad_norm': 0.9599031805992126, 'learning_rate': 0.0001955572805786141, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1336.48, 'epoch': 0.31}
16%|█████████████████████▍ | 7/44 [01:16<06:33, 10.64s/it]
18%|████████████████████████▌ | 8/44 [01:26<06:22, 10.62s/it]
{'loss': 0.0472, 'grad_norm': 0.6724708080291748, 'learning_rate': 0.00019308737486442045, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1326.52, 'epoch': 0.36}
18%|████████████████████████▌ | 8/44 [01:26<06:22, 10.62s/it]
20%|███████████████████████████▌ | 9/44 [01:37<06:10, 10.58s/it]
{'loss': 0.0535, 'grad_norm': 0.7262558937072754, 'learning_rate': 0.0001900968867902419, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1321.8, 'epoch': 0.4}
20%|███████████████████████████▌ | 9/44 [01:37<06:10, 10.58s/it]
23%|██████████████████████████████▍ | 10/44 [01:47<05:59, 10.58s/it]
{'loss': 0.0401, 'grad_norm': 0.5925001502037048, 'learning_rate': 0.00018660254037844388, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1341.09, 'epoch': 0.44}
23%|██████████████████████████████▍ | 10/44 [01:47<05:59, 10.58s/it]
25%|█████████████████████████████████▌ | 11/44 [01:58<05:48, 10.58s/it]
{'loss': 0.0421, 'grad_norm': 0.4922492504119873, 'learning_rate': 0.0001826238774315995, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1344.8, 'epoch': 0.49}
25%|█████████████████████████████████▌ | 11/44 [01:58<05:48, 10.58s/it]
27%|████████████████████████████████████▌ | 12/44 [02:09<05:38, 10.59s/it]
{'loss': 0.0496, 'grad_norm': 0.3521144390106201, 'learning_rate': 0.000178183148246803, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1236.47, 'epoch': 0.53}
27%|████████████████████████████████████▌ | 12/44 [02:09<05:38, 10.59s/it]
30%|███████████████████████████████████████▌ | 13/44 [02:19<05:28, 10.59s/it]
{'loss': 0.0422, 'grad_norm': 0.33798691630363464, 'learning_rate': 0.00017330518718298264, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1289.89, 'epoch': 0.58}
30%|███████████████████████████████████████▌ | 13/44 [02:19<05:28, 10.59s/it]
32%|██████████████████████████████████████████▋ | 14/44 [02:30<05:17, 10.60s/it]
{'loss': 0.0416, 'grad_norm': 0.33821234107017517, 'learning_rate': 0.00016801727377709194, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1264.76, 'epoch': 0.62}
32%|██████████████████████████████████████████▋ | 14/44 [02:30<05:17, 10.60s/it]
34%|█████████████████████████████████████████████▋ | 15/44 [02:40<05:07, 10.61s/it]
{'loss': 0.0283, 'grad_norm': 0.2641695737838745, 'learning_rate': 0.00016234898018587337, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1330.45, 'epoch': 0.67}
34%|█████████████████████████████████████████████▋ | 15/44 [02:40<05:07, 10.61s/it]
36%|████████████████████████████████████████████████▋ | 16/44 [02:51<04:56, 10.60s/it]
{'loss': 0.0426, 'grad_norm': 0.3202700912952423, 'learning_rate': 0.0001563320058063622, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1269.3, 'epoch': 0.71}
36%|████████████████████████████████████████████████▋ | 16/44 [02:51<04:56, 10.60s/it]
39%|███████████████████████████████████████████████████▊ | 17/44 [03:02<04:45, 10.59s/it]
{'loss': 0.0402, 'grad_norm': 0.37863489985466003, 'learning_rate': 0.00015000000000000001, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1276.11, 'epoch': 0.76}
39%|███████████████████████████████████████████████████▊ | 17/44 [03:02<04:45, 10.59s/it]
41%|██████████████████████████████████████████████████████▊ | 18/44 [03:12<04:35, 10.59s/it]
{'loss': 0.0541, 'grad_norm': 0.36693161725997925, 'learning_rate': 0.00014338837391175582, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1293.35, 'epoch': 0.8}
41%|██████████████████████████████████████████████████████▊ | 18/44 [03:12<04:35, 10.59s/it]
43%|█████████████████████████████████████████████████████████▊ | 19/44 [03:23<04:25, 10.60s/it]
{'loss': 0.0513, 'grad_norm': 0.398545503616333, 'learning_rate': 0.00013653410243663952, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1218.05, 'epoch': 0.84}
43%|█████████████████████████████████████████████████████████▊ | 19/44 [03:23<04:25, 10.60s/it]
45%|████████████████████████████████████████████████████████████▉ | 20/44 [03:33<04:14, 10.60s/it]
{'loss': 0.0283, 'grad_norm': 0.24128811061382294, 'learning_rate': 0.00012947551744109043, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1243.65, 'epoch': 0.89}
45%|████████████████████████████████████████████████████████████▉ | 20/44 [03:33<04:14, 10.60s/it]
48%|███████████████████████████████████████████████████████████████▉ | 21/44 [03:44<04:03, 10.57s/it]
{'loss': 0.0238, 'grad_norm': 0.22816111147403717, 'learning_rate': 0.00012225209339563145, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1379.37, 'epoch': 0.93}
48%|███████████████████████████████████████████████████████████████▉ | 21/44 [03:44<04:03, 10.57s/it]
50%|███████████████████████████████████████████████████████████████████ | 22/44 [03:54<03:52, 10.57s/it]
{'loss': 0.0272, 'grad_norm': 0.29317620396614075, 'learning_rate': 0.00011490422661761744, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1257.24, 'epoch': 0.98}
50%|███████████████████████████████████████████████████████████████████ | 22/44 [03:54<03:52, 10.57s/it]
52%|██████████████████████████████████████████████████████████████████████ | 23/44 [04:00<03:09, 9.01s/it]
{'loss': 0.0446, 'grad_norm': 0.49180206656455994, 'learning_rate': 0.00010747300935864243, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1165.84, 'epoch': 1.0}
52%|██████████████████████████████████████████████████████████████████████ | 23/44 [04:00<03:09, 9.01s/it]
55%|█████████████████████████████████████████████████████████████████████████ | 24/44 [04:12<03:21, 10.07s/it]
{'loss': 0.0162, 'grad_norm': 0.23574988543987274, 'learning_rate': 0.0001, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1261.03, 'epoch': 1.04}
55%|█████████████████████████████████████████████████████████████████████████ | 24/44 [04:12<03:21, 10.07s/it]
57%|████████████████████████████████████████████████████████████████████████████▏ | 25/44 [04:23<03:13, 10.20s/it]
{'loss': 0.0174, 'grad_norm': 0.17455901205539703, 'learning_rate': 9.252699064135758e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1388.76, 'epoch': 1.09}
57%|████████████████████████████████████████████████████████████████████████████▏ | 25/44 [04:23<03:13, 10.20s/it]
59%|███████████████████████████████████████████████████████████████████████████████▏ | 26/44 [04:33<03:05, 10.32s/it]
{'loss': 0.0185, 'grad_norm': 0.23850183188915253, 'learning_rate': 8.509577338238255e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1238.14, 'epoch': 1.13}
59%|███████████████████████████████████████████████████████████████████████████████▏ | 26/44 [04:33<03:05, 10.32s/it]
61%|██████████████████████████████████████████████████████████████████████████████████▏ | 27/44 [04:44<02:56, 10.39s/it]
{'loss': 0.0137, 'grad_norm': 0.17631582915782928, 'learning_rate': 7.774790660436858e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1331.49, 'epoch': 1.18}
61%|██████████████████████████████████████████████████████████████████████████████████▏ | 27/44 [04:44<02:56, 10.39s/it]
64%|█████████████████████████████████████████████████████████████████████████████████████▎ | 28/44 [04:55<02:46, 10.43s/it]
{'loss': 0.0133, 'grad_norm': 0.1664351224899292, 'learning_rate': 7.052448255890957e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1321.29, 'epoch': 1.22}
64%|█████████████████████████████████████████████████████████████████████████████████████▎ | 28/44 [04:55<02:46, 10.43s/it]
66%|████████████████████████████████████████████████████████████████████████████████████████▎ | 29/44 [05:05<02:37, 10.48s/it]
{'loss': 0.0153, 'grad_norm': 0.20239098370075226, 'learning_rate': 6.34658975633605e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1232.78, 'epoch': 1.27}
66%|████████████████████████████████████████████████████████████████████████████████████████▎ | 29/44 [05:05<02:37, 10.48s/it]
68%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 30/44 [05:16<02:27, 10.52s/it]
{'loss': 0.0197, 'grad_norm': 0.20838677883148193, 'learning_rate': 5.6611626088244194e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1195.49, 'epoch': 1.31}
68%|███████████████████████████████████████████████████████████████████████████████████████████▎ | 30/44 [05:16<02:27, 10.52s/it]
70%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 31/44 [05:26<02:17, 10.55s/it]
{'loss': 0.0161, 'grad_norm': 0.17082557082176208, 'learning_rate': 5.000000000000002e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1237.56, 'epoch': 1.36}
70%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 31/44 [05:26<02:17, 10.55s/it]
73%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32/44 [05:37<02:06, 10.55s/it]
{'loss': 0.0126, 'grad_norm': 0.17102588713169098, 'learning_rate': 4.3667994193637796e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1292.78, 'epoch': 1.4}
73%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 32/44 [05:37<02:06, 10.55s/it]
75%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33/44 [05:48<01:56, 10.56s/it]
{'loss': 0.0155, 'grad_norm': 0.17077268660068512, 'learning_rate': 3.7651019814126654e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1433.05, 'epoch': 1.44}
75%|████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 33/44 [05:48<01:56, 10.56s/it]
77%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34/44 [05:58<01:45, 10.55s/it]
{'loss': 0.0125, 'grad_norm': 0.1558006852865219, 'learning_rate': 3.198272622290804e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1341.43, 'epoch': 1.49}
77%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 34/44 [05:58<01:45, 10.55s/it]
80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35/44 [06:09<01:34, 10.53s/it]
{'loss': 0.0121, 'grad_norm': 0.17341959476470947, 'learning_rate': 2.669481281701739e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1390.06, 'epoch': 1.53}
80%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 35/44 [06:09<01:34, 10.53s/it]
82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36/44 [06:19<01:24, 10.53s/it]
{'loss': 0.0135, 'grad_norm': 0.7362567186355591, 'learning_rate': 2.181685175319702e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1381.11, 'epoch': 1.58}
82%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 36/44 [06:19<01:24, 10.53s/it]
84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37/44 [06:30<01:13, 10.53s/it]
{'loss': 0.0126, 'grad_norm': 0.20648105442523956, 'learning_rate': 1.7376122568400532e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1296.05, 'epoch': 1.62}
84%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 37/44 [06:30<01:13, 10.53s/it]
86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38/44 [06:40<01:03, 10.54s/it]
{'loss': 0.0149, 'grad_norm': 0.17279624938964844, 'learning_rate': 1.339745962155613e-05, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1320.12, 'epoch': 1.67}
86%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 38/44 [06:40<01:03, 10.54s/it]
89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39/44 [06:51<00:52, 10.54s/it]
{'loss': 0.0113, 'grad_norm': 0.1708693951368332, 'learning_rate': 9.903113209758096e-06, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1333.73, 'epoch': 1.71}
89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 39/44 [06:51<00:52, 10.54s/it]
91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40/44 [07:01<00:42, 10.57s/it]
{'loss': 0.0119, 'grad_norm': 0.15244145691394806, 'learning_rate': 6.9126251355795864e-06, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1278.44, 'epoch': 1.76}
91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 40/44 [07:01<00:42, 10.57s/it]
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 41/44 [07:12<00:31, 10.57s/it]
{'loss': 0.0121, 'grad_norm': 0.15926013886928558, 'learning_rate': 4.442719421385922e-06, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1296.28, 'epoch': 1.8}
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 41/44 [07:12<00:31, 10.57s/it]
95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42/44 [07:22<00:21, 10.58s/it]
{'loss': 0.0186, 'grad_norm': 0.20554262399673462, 'learning_rate': 2.5072087818176382e-06, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1198.58, 'epoch': 1.84}
95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 42/44 [07:23<00:21, 10.58s/it]
98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43/44 [07:33<00:10, 10.59s/it]
{'loss': 0.02, 'grad_norm': 0.23918209969997406, 'learning_rate': 1.1169173774871478e-06, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1198.53, 'epoch': 1.89}
98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 43/44 [07:33<00:10, 10.59s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 44/44 [07:44<00:00, 10.58s/it]
{'loss': 0.0093, 'grad_norm': 0.1438334584236145, 'learning_rate': 2.7962028188198706e-07, 'memory/max_active (GiB)': 43.35, 'memory/max_allocated (GiB)': 43.35, 'memory/device_reserved (GiB)': 51.13, 'tokens_per_second_per_gpu': 1367.42, 'epoch': 1.93}
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 44/44 [07:44<00:00, 10.58s/it][2026-04-10 14:06:46,792] [INFO] [axolotl.core.trainers.base._save:671] [PID:7899] Saving model checkpoint to ./outputs/mymodel/checkpoint-44
{'train_runtime': 472.0616, 'train_samples_per_second': 1.491, 'train_steps_per_second': 0.093, 'train_loss': 0.0675379387949678, 'memory/max_active (GiB)': 12.66, 'memory/max_allocated (GiB)': 12.66, 'memory/device_reserved (GiB)': 51.13, 'epoch': 1.93}
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 44/44 [07:52<00:00, 10.58s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 44/44 [07:52<00:00, 10.73s/it]
[2026-04-10 14:06:54,929] [INFO] [axolotl.train.save_trained_model:218] [PID:7899] Training completed! Saving trained model to ./outputs/mymodel.
[2026-04-10 14:06:58,057] [INFO] [axolotl.train.save_trained_model:336] [PID:7899] Model successfully saved to ./outputs/mymodel
|