File size: 30,294 Bytes
ca3c54a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 |
[2026-01-25 09:34:01,866] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:10047] baseline 0.000GB ()
[2026-01-25 09:34:01,867] [INFO] [axolotl.cli.config.load_cfg:259] [PID:10047] config:
{
"activation_offloading": false,
"axolotl_config_path": "train.yml",
"base_model": "google/gemma-3-4b-it",
"base_model_config": "google/gemma-3-4b-it",
"batch_size": 13,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_86",
"fp8": false,
"n_gpu": 1,
"n_node": 1
},
"context_parallel_size": 1,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_num_proc": 9,
"datasets": [
{
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "AlexHung29629/MerlynIfeEldridge2",
"trust_remote_code": false,
"type": "input_output"
}
],
"ddp": false,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"env_capabilities": {
"torch_version": "2.9.1"
},
"eval_batch_size": 13,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"fp16": false,
"gradient_accumulation_steps": 1,
"gradient_checkpointing": true,
"gradient_checkpointing_kwargs": {
"use_reentrant": false
},
"include_tkps": true,
"is_multimodal": true,
"learning_rate": 0.001,
"liger_fused_linear_cross_entropy": true,
"liger_glu_activation": true,
"liger_layer_norm": true,
"liger_rms_norm": true,
"liger_rope": true,
"liger_use_token_scaling": true,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": false,
"load_in_8bit": false,
"local_rank": 0,
"lora_dropout": 0.0,
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "constant",
"max_grad_norm": 1.0,
"mean_resizing_embeddings": false,
"micro_batch_size": 13,
"model_config_type": "gemma3",
"num_epochs": 16.0,
"optimizer": "sgd",
"otel_metrics_host": "localhost",
"otel_metrics_port": 8000,
"output_dir": "./model-out",
"plugins": [
"axolotl.integrations.liger.LigerPlugin"
],
"pretrain_multipack_attn": true,
"processor_config": "google/gemma-3-4b-it",
"profiler_steps_start": 0,
"qlora_sharded_model_loading": false,
"ray_num_workers": 1,
"resources_per_worker": {
"GPU": 1
},
"sample_packing": false,
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": false,
"save_safetensors": true,
"save_strategy": "no",
"seed": 42,
"sequence_len": 758,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"tf32": true,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "google/gemma-3-4b-it",
"tokenizer_save_jinja_files": true,
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"log_completions": false,
"mask_truncated_completions": false,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"scale_rewards": true,
"sync_ref_model": false,
"use_vllm": false,
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"use_otel_metrics": false,
"use_ray": false,
"use_tensorboard": true,
"use_wandb": false,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"gpu_memory_utilization": 0.9,
"host": "0.0.0.0",
"port": 8000
},
"warmup_ratio": 0.0,
"weight_decay": 0.0,
"world_size": 1
}
[2026-01-25 09:34:01,983] [DEBUG] [axolotl.loaders.utils.check_model_config:88] [PID:10047] Loaded image size: 896 from model config
[2026-01-25 09:34:03,852] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:10047] EOS: 1 / <eos>
[2026-01-25 09:34:03,853] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:10047] BOS: 2 / <bos>
[2026-01-25 09:34:03,853] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:10047] PAD: 0 / <pad>
[2026-01-25 09:34:03,853] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:10047] UNK: 3 / <unk>
[2026-01-25 09:34:03,853] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:10047] Unable to find prepared dataset in last_run_prepared/79c123e6ef0babe72cf6db37825069f8
[2026-01-25 09:34:03,854] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:10047] Loading raw datasets...
[2026-01-25 09:34:03,854] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:10047] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
[2026-01-25 09:34:08,929] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:10047] Loading dataset: AlexHung29629/MerlynIfeEldridge2 with base_type: input_output and prompt_style: None
[2026-01-25 09:34:09,525] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:224] [PID:10047] min_input_len: 152
[2026-01-25 09:34:09,526] [INFO] [axolotl.utils.data.utils.handle_long_seq_in_dataset:226] [PID:10047] max_input_len: 676
Saving the dataset (0/1 shards): 0%| | 0/13 [00:00<?, ? examples/s]
Saving the dataset (0/1 shards): 100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 59.57 examples/s]
Saving the dataset (1/1 shards): 100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 59.57 examples/s]
Saving the dataset (1/1 shards): 100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:00<00:00, 41.66 examples/s]
[2026-01-25 09:34:10,159] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:417] [PID:10047] total_num_tokens: 4_827
[2026-01-25 09:34:10,162] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:435] [PID:10047] `total_supervised_tokens: 43`
[2026-01-25 09:34:10,162] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:533] [PID:10047] total_num_steps: 16
[2026-01-25 09:34:10,163] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:10047] Maximum number of steps set at 16
[2026-01-25 09:34:10,283] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:10047] loading tokenizer... google/gemma-3-4b-it
[2026-01-25 09:34:12,371] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:10047] EOS: 1 / <eos>
[2026-01-25 09:34:12,371] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:10047] BOS: 2 / <bos>
[2026-01-25 09:34:12,371] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:10047] PAD: 0 / <pad>
[2026-01-25 09:34:12,371] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:10047] UNK: 3 / <unk>
[2026-01-25 09:34:23,539] [DEBUG] [axolotl.train.setup_model_and_tokenizer:82] [PID:10047] Loading model
[2026-01-25 09:34:23,660] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:10047] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-01-25 09:34:23,662] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:10047] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-01-25 09:34:23,793] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:98] [PID:10047] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': True, 'geglu': True}
Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards: 50%|████████████████████████████████████████████████████████▌ | 1/2 [00:01<00:01, 1.58s/it]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:02<00:00, 1.18s/it]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:02<00:00, 1.24s/it]
[2026-01-25 09:34:32,161] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:10047] Memory usage after model load 0.000GB ()
[2026-01-25 09:35:21,180] [INFO] [axolotl.train.save_initial_configs:417] [PID:10047] Pre-saving tokenizer to ./model-out...
[2026-01-25 09:35:21,802] [INFO] [axolotl.train.save_initial_configs:422] [PID:10047] Pre-saving model config to ./model-out...
[2026-01-25 09:35:21,809] [INFO] [axolotl.train.save_initial_configs:426] [PID:10047] Pre-saving processor to ./model-out...
[2026-01-25 09:35:25,524] [INFO] [axolotl.train.execute_training:212] [PID:10047] Starting trainer...
0%| | 0/16 [00:00<?, ?it/s]
6%|████████▋ | 1/16 [00:07<01:45, 7.05s/it]
{'loss': 0.0345, 'grad_norm': 61.53063201904297, 'learning_rate': 0.001, 'ppl': 1.0351, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 6.187943458557129, 'tokens/total': 9152, 'tokens/trainable': 43, 'epoch': 1.0}
6%|████████▋ | 1/16 [00:07<01:45, 7.05s/it]
12%|█████████████████▍ | 2/16 [00:13<01:31, 6.54s/it]
{'loss': 0.033, 'grad_norm': 57.19621276855469, 'learning_rate': 0.001, 'ppl': 1.03355, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.120673656463623, 'tokens/total': 18304, 'tokens/trainable': 86, 'epoch': 2.0}
12%|█████████████████▍ | 2/16 [00:13<01:31, 6.54s/it]
19%|██████████████████████████ | 3/16 [00:19<01:23, 6.39s/it]
{'loss': 0.0321, 'grad_norm': 57.623077392578125, 'learning_rate': 0.001, 'ppl': 1.03262, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.125824928283691, 'tokens/total': 27456, 'tokens/trainable': 129, 'epoch': 3.0}
19%|██████████████████████████ | 3/16 [00:19<01:23, 6.39s/it]
25%|██████████████████████████████████▊ | 4/16 [00:25<01:15, 6.33s/it]
{'loss': 0.0299, 'grad_norm': 63.824161529541016, 'learning_rate': 0.001, 'ppl': 1.03035, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.115628719329834, 'tokens/total': 36608, 'tokens/trainable': 172, 'epoch': 4.0}
25%|██████████████████████████████████▊ | 4/16 [00:25<01:15, 6.33s/it]
31%|███████████████████████████████████████████▍ | 5/16 [00:31<01:09, 6.29s/it]
{'loss': 0.03, 'grad_norm': 61.47892761230469, 'learning_rate': 0.001, 'ppl': 1.03045, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.117379665374756, 'tokens/total': 45760, 'tokens/trainable': 215, 'epoch': 5.0}
31%|███████████████████████████████████████████▍ | 5/16 [00:32<01:09, 6.29s/it]
38%|████████████████████████████████████████████████████▏ | 6/16 [00:38<01:02, 6.27s/it]
{'loss': 0.0242, 'grad_norm': 40.61567687988281, 'learning_rate': 0.001, 'ppl': 1.0245, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.108977317810059, 'tokens/total': 54912, 'tokens/trainable': 258, 'epoch': 6.0}
38%|████████████████████████████████████████████████████▏ | 6/16 [00:38<01:02, 6.27s/it]
44%|████████████████████████████████████████████████████████████▊ | 7/16 [00:44<00:56, 6.26s/it]
{'loss': 0.0225, 'grad_norm': 31.520526885986328, 'learning_rate': 0.001, 'ppl': 1.02276, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.105500221252441, 'tokens/total': 64064, 'tokens/trainable': 301, 'epoch': 7.0}
44%|████████████████████████████████████████████████████████████▊ | 7/16 [00:44<00:56, 6.26s/it]
50%|█████████████████████████████████████████████████████████████████████▌ | 8/16 [00:50<00:50, 6.26s/it]
{'loss': 0.0217, 'grad_norm': 29.32663917541504, 'learning_rate': 0.001, 'ppl': 1.02194, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.099156379699707, 'tokens/total': 73216, 'tokens/trainable': 344, 'epoch': 8.0}
50%|█████████████████████████████████████████████████████████████████████▌ | 8/16 [00:50<00:50, 6.26s/it]
56%|██████████████████████████████████████████████████████████████████████████████▏ | 9/16 [00:56<00:43, 6.26s/it]
{'loss': 0.0211, 'grad_norm': 26.701892852783203, 'learning_rate': 0.001, 'ppl': 1.02132, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.0905585289001465, 'tokens/total': 82368, 'tokens/trainable': 387, 'epoch': 9.0}
56%|██████████████████████████████████████████████████████████████████████████████▏ | 9/16 [00:56<00:43, 6.26s/it]
62%|██████████████████████████████████████████████████████████████████████████████████████▎ | 10/16 [01:03<00:37, 6.26s/it]
{'loss': 0.0205, 'grad_norm': 24.277631759643555, 'learning_rate': 0.001, 'ppl': 1.02071, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.076664447784424, 'tokens/total': 91520, 'tokens/trainable': 430, 'epoch': 10.0}
62%|██████████████████████████████████████████████████████████████████████████████████████▎ | 10/16 [01:03<00:37, 6.26s/it]
69%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 11/16 [01:09<00:31, 6.26s/it]
{'loss': 0.02, 'grad_norm': 24.709354400634766, 'learning_rate': 0.001, 'ppl': 1.0202, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.076472759246826, 'tokens/total': 100672, 'tokens/trainable': 473, 'epoch': 11.0}
69%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 11/16 [01:09<00:31, 6.26s/it]
75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 12/16 [01:15<00:25, 6.27s/it]
{'loss': 0.0187, 'grad_norm': 23.36050033569336, 'learning_rate': 0.001, 'ppl': 1.01888, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.066161155700684, 'tokens/total': 109824, 'tokens/trainable': 516, 'epoch': 12.0}
75%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 12/16 [01:15<00:25, 6.27s/it]
81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 13/16 [01:21<00:18, 6.27s/it]
{'loss': 0.0187, 'grad_norm': 25.07172393798828, 'learning_rate': 0.001, 'ppl': 1.01888, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.074800968170166, 'tokens/total': 118976, 'tokens/trainable': 559, 'epoch': 13.0}
81%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 13/16 [01:22<00:18, 6.27s/it]
88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 14/16 [01:28<00:12, 6.27s/it]
{'loss': 0.0172, 'grad_norm': 24.219331741333008, 'learning_rate': 0.001, 'ppl': 1.01735, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.072431564331055, 'tokens/total': 128128, 'tokens/trainable': 602, 'epoch': 14.0}
88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 14/16 [01:28<00:12, 6.27s/it]
94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15/16 [01:34<00:06, 6.26s/it]
{'loss': 0.0166, 'grad_norm': 23.965293884277344, 'learning_rate': 0.001, 'ppl': 1.01674, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.07435417175293, 'tokens/total': 137280, 'tokens/trainable': 645, 'epoch': 15.0}
94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 15/16 [01:34<00:06, 6.26s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:40<00:00, 6.27s/it]
{'loss': 0.0139, 'grad_norm': 21.725933074951172, 'learning_rate': 0.001, 'ppl': 1.014, 'memory/max_active (GiB)': 20.46, 'memory/max_allocated (GiB)': 20.46, 'memory/device_reserved (GiB)': 23.4, 'tokens/train_per_sec_per_gpu': 7.0699357986450195, 'tokens/total': 146432, 'tokens/trainable': 688, 'epoch': 16.0}
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:40<00:00, 6.27s/it]
{'train_runtime': 100.8727, 'train_samples_per_second': 2.062, 'train_steps_per_second': 0.159, 'train_loss': 0.023414524795953184, 'memory/max_active (GiB)': 9.29, 'memory/max_allocated (GiB)': 9.29, 'memory/device_reserved (GiB)': 23.4, 'epoch': 16.0, 'tokens/train_per_sec_per_gpu': 0.0}
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:40<00:00, 6.27s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [01:40<00:00, 6.30s/it]
[2026-01-25 09:37:06,886] [INFO] [axolotl.train.save_trained_model:233] [PID:10047] Training completed! Saving trained model to ./model-out.
[2026-01-25 09:37:19,416] [INFO] [axolotl.train.save_trained_model:351] [PID:10047] Model successfully saved to ./model-out
|