| [2026-05-16 18:20:43,308] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:14603] bf16 support detected, enabling for this configuration. |
| [2026-05-16 18:20:43,452] [WARNING] [huggingface_hub.utils._http._warn_on_warning_headers:904] [PID:14603] Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. |
| [2026-05-16 18:20:43,465] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:14603] baseline 0.000GB () |
| [2026-05-16 18:20:43,467] [INFO] [axolotl.cli.config.load_cfg:333] [PID:14603] config: |
| { |
| "activation_offloading": true, |
| "attn_implementation": "flash_attention_2", |
| "attn_needs_dtype_cast": true, |
| "attn_supports_packing": true, |
| "attn_uses_flash_lib": true, |
| "axolotl_config_path": "train.yml", |
| "base_model": "shb777/Llama-3.3-8B-Instruct-128K", |
| "base_model_config": "shb777/Llama-3.3-8B-Instruct-128K", |
| "batch_size": 32, |
| "bf16": true, |
| "capabilities": { |
| "bf16": true, |
| "compute_capability": "sm_80", |
| "fp8": false, |
| "n_gpu": 1, |
| "n_node": 1, |
| "tf32": true |
| }, |
| "chat_template": "tokenizer_default", |
| "context_parallel_size": 1, |
| "cosine_min_lr_ratio": 0.1, |
| "cut_cross_entropy": true, |
| "dataloader_num_workers": 1, |
| "dataloader_pin_memory": true, |
| "dataloader_prefetch_factor": 256, |
| "dataset_num_proc": 32, |
| "dataset_prepared_path": "last_run_prepared", |
| "datasets": [ |
| { |
| "ds_type": "parquet", |
| "message_property_mappings": { |
| "content": "content", |
| "role": "role" |
| }, |
| "path": "tankieV2-llama3.3.parquet", |
| "trust_remote_code": false |
| } |
| ], |
| "ddp": false, |
| "device": "cuda:0", |
| "dion_rank_fraction": 1.0, |
| "dion_rank_multiple_of": 1, |
| "eaft_alpha": 1.0, |
| "eaft_k": 20, |
| "env_capabilities": { |
| "torch_version": "2.8.0" |
| }, |
| "eval_batch_size": 4, |
| "eval_causal_lm_metrics": [ |
| "sacrebleu", |
| "comet", |
| "ter", |
| "chrf" |
| ], |
| "eval_max_new_tokens": 128, |
| "eval_sample_packing": true, |
| "eval_table_size": 0, |
| "experimental_skip_move_to_device": true, |
| "fp16": false, |
| "generate_samples": false, |
| "generation_do_sample": true, |
| "generation_max_new_tokens": 50, |
| "generation_prompt_ratio": 0.5, |
| "generation_temperature": 0.7, |
| "gradient_accumulation_steps": 8, |
| "gradient_checkpointing": true, |
| "gradient_checkpointing_kwargs": { |
| "use_reentrant": true |
| }, |
| "group_by_length": false, |
| "include_tkps": true, |
| "is_llama_derived_model": true, |
| "layer_offloading": false, |
| "learning_rate": 5e-06, |
| "lisa_layers_attribute": "model.layers", |
| "load_best_model_at_end": false, |
| "load_in_4bit": false, |
| "load_in_8bit": false, |
| "local_rank": 0, |
| "logging_steps": 1, |
| "lora_dropout": 0.0, |
| "loraplus_lr_embedding": 1e-06, |
| "lr_scheduler": "constant_with_warmup", |
| "max_grad_norm": 0.1, |
| "mean_resizing_embeddings": false, |
| "merge_method": "memory_efficient", |
| "micro_batch_size": 4, |
| "model_config_type": "llama", |
| "num_epochs": 3.0, |
| "num_generation_samples": 3, |
| "optimizer": "adamw_torch_fused", |
| "otel_metrics_host": "localhost", |
| "otel_metrics_port": 8000, |
| "output_dir": "./model-output", |
| "pad_to_sequence_len": true, |
| "plugins": [ |
| "axolotl.integrations.liger.LigerPlugin", |
| "axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin" |
| ], |
| "pretrain_multipack_attn": true, |
| "profiler_steps_start": 0, |
| "qlora_sharded_model_loading": false, |
| "quantize_moe_experts": false, |
| "ray_num_workers": 1, |
| "relora_prune_method": "magnitude", |
| "resources_per_worker": { |
| "GPU": 1 |
| }, |
| "sample_packing": true, |
| "sample_packing_bin_size": 200, |
| "sample_packing_group_size": 100000, |
| "save_only_model": false, |
| "save_safetensors": true, |
| "saves_per_epoch": 0, |
| "sequence_len": 4096, |
| "shuffle_before_merging_datasets": false, |
| "shuffle_merged_datasets": true, |
| "skip_prepare_dataset": false, |
| "streaming_multipack_buffer_size": 10000, |
| "strict": false, |
| "tensor_parallel_size": 1, |
| "tf32": true, |
| "tiled_mlp_use_original_mlp": true, |
| "tokenizer_config": "shb777/Llama-3.3-8B-Instruct-128K", |
| "tokenizer_save_jinja_files": true, |
| "torch_dtype": "torch.bfloat16", |
| "train_on_inputs": false, |
| "trl": { |
| "async_prefetch": false, |
| "log_completions": false, |
| "mask_truncated_completions": false, |
| "ref_model_mixup_alpha": 0.9, |
| "ref_model_sync_steps": 64, |
| "replay_buffer_size": 0, |
| "replay_recompute_logps": true, |
| "reroll_max_groups": 1, |
| "reroll_start_fraction": 1.0, |
| "reward_num_workers": 1, |
| "scale_rewards": true, |
| "skip_zero_advantage_batches": true, |
| "sync_ref_model": false, |
| "use_data_producer": false, |
| "use_vllm": false, |
| "vllm_lora_sync": false, |
| "vllm_server_host": "0.0.0.0", |
| "vllm_server_port": 8000 |
| }, |
| "trust_remote_code": false, |
| "use_otel_metrics": false, |
| "use_ray": false, |
| "use_wandb": true, |
| "val_set_size": 0.0, |
| "vllm": { |
| "device": "auto", |
| "dtype": "auto", |
| "gpu_memory_utilization": 0.9, |
| "host": "0.0.0.0", |
| "port": 8000 |
| }, |
| "wandb_project": "tankieV2-llama3.3", |
| "warmup_ratio": 0.05, |
| "weight_decay": 0.001, |
| "world_size": 1 |
| } |
| [2026-05-16 18:20:43,470] [WARNING] [axolotl.cli.checks.check_user_token:48] [PID:14603] Error verifying HuggingFace token. Remember to log in using `hf auth login` and get your access token from https://huggingface.co/settings/tokens if you want to use gated models or datasets. |
| [2026-05-16 18:20:45,130] [WARNING] [axolotl.loaders.tokenizer.load_tokenizer:305] [PID:14603] Tokenizer does not have a pad_token, falling back to eos_token: <|eot_id|> |
| [2026-05-16 18:20:45,131] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:14603] EOS: 128009 / <|eot_id|> |
| [2026-05-16 18:20:45,131] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:14603] BOS: 128000 / <|begin_of_text|> |
| [2026-05-16 18:20:45,131] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:14603] PAD: 128009 / <|eot_id|> |
| [2026-05-16 18:20:45,132] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:14603] UNK: None / None |
| [2026-05-16 18:20:45,133] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:477] [PID:14603] Loading prepared dataset from disk at last_run_prepared/8e4debe9138a048d05e5d68d1585e372... |
| [2026-05-16 18:20:45,154] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:14603] total_num_tokens: 1_063_789 |
| [2026-05-16 18:20:45,170] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:14603] `total_supervised_tokens: 884_389` |
| [2026-05-16 18:20:47,287] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.073195457458496 |
| [2026-05-16 18:20:48,338] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0501418113708496 |
| [2026-05-16 18:20:49,358] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0195682048797607 |
| [2026-05-16 18:20:50,431] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0725367069244385 |
| [2026-05-16 18:20:50,460] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:14603] gather_len_batches: [66] |
| [2026-05-16 18:20:50,460] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:495] [PID:14603] data_loader_len: 8 |
| [2026-05-16 18:20:50,460] [INFO] [axolotl.utils.trainer.calc_sample_packing_eff_est:504] [PID:14603] sample_packing_eff_est across ranks: [0.9837655732125947] |
| [2026-05-16 18:20:50,460] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:516] [PID:14603] sample_packing_eff_est: 0.99 |
| [2026-05-16 18:20:50,460] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:14603] total_num_steps: 24 |
| [2026-05-16 18:20:50,461] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:14603] Maximum number of steps set at 24 |
| [2026-05-16 18:20:50,514] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:14603] loading tokenizer... shb777/Llama-3.3-8B-Instruct-128K |
| [2026-05-16 18:20:52,167] [WARNING] [axolotl.loaders.tokenizer.load_tokenizer:305] [PID:14603] Tokenizer does not have a pad_token, falling back to eos_token: <|eot_id|> |
| [2026-05-16 18:20:52,167] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:311] [PID:14603] EOS: 128009 / <|eot_id|> |
| [2026-05-16 18:20:52,168] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:312] [PID:14603] BOS: 128000 / <|begin_of_text|> |
| [2026-05-16 18:20:52,168] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:313] [PID:14603] PAD: 128009 / <|eot_id|> |
| [2026-05-16 18:20:52,168] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:314] [PID:14603] UNK: None / None |
| [2026-05-16 18:20:52,168] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:14603] Loading model |
| [2026-05-16 18:20:52,254] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:14603] Patched OptimState8bit for torch.compile compatibility |
| [2026-05-16 18:20:52,254] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:14603] Patched OptimState4bit for torch.compile compatibility |
| [2026-05-16 18:20:52,255] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:14603] Patched OptimStateFp8 for torch.compile compatibility |
| [2026-05-16 18:20:52,262] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:14603] Patched Trainer.evaluation_loop with nanmean loss calculation |
| [2026-05-16 18:20:52,264] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:14603] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation |
| [2026-05-16 18:20:52,265] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:598] [PID:14603] Applying multipack dataloader patch for sample packing... |
| [2026-05-16 18:20:52,504] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:104] [PID:14603] Applying LIGER to llama with kwargs: {'rope': None, 'cross_entropy': None, 'fused_linear_cross_entropy': None, 'rms_norm': None, 'swiglu': None} |
| [2026-05-16 18:20:52,534] [INFO] [axolotl.integrations.cut_cross_entropy.pre_model_load:94] [PID:14603] Applying Cut Cross Entropy to model type: llama |
|
Loading weights: 0%| | 0/291 [00:00<?, ?it/s]
Loading weights: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 291/291 [00:00<00:00, 6604.60it/s] |
| [2026-05-16 18:20:55,713] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:356] [PID:14603] Converting modules to torch.bfloat16 |
| [2026-05-16 18:20:57,542] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:14603] Memory usage after model load 0.000GB () |
| [2026-05-16 18:21:01,654] [INFO] [axolotl.train.save_initial_configs:454] [PID:14603] Pre-saving tokenizer to ./model-output... |
| [2026-05-16 18:21:01,958] [INFO] [axolotl.train.save_initial_configs:459] [PID:14603] Pre-saving model config to ./model-output... |
| [2026-05-16 18:21:01,964] [INFO] [axolotl.train.execute_training:226] [PID:14603] Starting trainer... |
| [2026-05-16 18:21:04,415] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0974228382110596 |
| [2026-05-16 18:21:05,443] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 1.0277769565582275 |
| [2026-05-16 18:21:06,430] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 0.9865751266479492 |
| [2026-05-16 18:21:07,419] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:14603] generate_batches time: 0.9884381294250488 |
| [2026-05-16 18:21:07,419] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:14603] gather_len_batches: [66] |
| [34m[1mwandb[0m: (1) Create a W&B account |
| [34m[1mwandb[0m: (2) Use an existing W&B account |
| [34m[1mwandb[0m: (3) Don't visualize my results |
| [34m[1mwandb[0m: Enter your choice:[34m[1mwandb[0m: You chose "Don't visualize my results" |
| [34m[1mwandb[0m: Using W&B in offline mode. |
| [34m[1mwandb[0m: W&B API key is configured. Use [1m`wandb login --relogin`[0m to force relogin |
| [34m[1mwandb[0m: Tracking run with wandb version 0.27.0 |
| [34m[1mwandb[0m: W&B syncing is set to [1m`offline`[0m in this directory. Run [1m`wandb online`[0m or set [1mWANDB_MODE=online[0m to enable cloud syncing. |
| [34m[1mwandb[0m: Run data is saved locally in [35m[1m/root/axolotl/wandb/offline-run-20260516_182119-8b276ifr[0m |
| [34m[1mwandb[0m: [33mWARNING[0m Saving files without folders. If you want to preserve subdirectories pass base_path to wandb.save, i.e. wandb.save("/mnt/folder/file.h5", base_path="/mnt") |
| [34m[1mwandb[0m: [33mWARNING[0m Symlinked 1 file into the W&B run directory; call wandb.save again to sync new files. |
| [2026-05-16 18:21:20,370] [INFO] [axolotl.utils.callbacks.on_train_begin:807] [PID:14603] The Axolotl config has been saved to the WandB run under files. |
|
0%| | 0/24 [00:00<?, ?it/s]
4%|███████▎ | 1/24 [00:43<16:41, 43.54s/it]
{'loss': '2.758', 'grad_norm': '46.75', 'learning_rate': '0', 'ppl': '15.77', 'memory/max_active (GiB)': '60.02', 'memory/max_allocated (GiB)': '60.02', 'memory/device_reserved (GiB)': '60.44', 'tokens/train_per_sec_per_gpu': '343.8', 'tokens/total': 131072, 'tokens/trainable': 109999, 'epoch': '0.1212'} |
|
4%|███████▎ | 1/24 [00:43<16:41, 43.54s/it]
8%|██████████████▌ | 2/24 [01:21<14:46, 40.28s/it]
{'loss': '2.747', 'grad_norm': '50.75', 'learning_rate': '2.5e-06', 'ppl': '15.6', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '337.1', 'tokens/total': 262144, 'tokens/trainable': 221663, 'epoch': '0.2424'} |
|
8%|██████████████▌ | 2/24 [01:21<14:46, 40.28s/it]
12%|█████████████████████▊ | 3/24 [01:59<13:45, 39.31s/it]
{'loss': '2.587', 'grad_norm': '33', 'learning_rate': '5e-06', 'ppl': '13.29', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '374.8', 'tokens/total': 393216, 'tokens/trainable': 333227, 'epoch': '0.3636'} |
|
12%|█████████████████████▊ | 3/24 [01:59<13:45, 39.31s/it]
17%|█████████████████████████████ | 4/24 [02:37<12:57, 38.86s/it]
{'loss': '2.485', 'grad_norm': '15.38', 'learning_rate': '5e-06', 'ppl': '12.01', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '339.4', 'tokens/total': 524288, 'tokens/trainable': 442213, 'epoch': '0.4848'} |
|
17%|█████████████████████████████ | 4/24 [02:37<12:57, 38.86s/it]
21%|████████████████████████████████████▎ | 5/24 [03:16<12:14, 38.66s/it]
{'loss': '2.396', 'grad_norm': '9.188', 'learning_rate': '5e-06', 'ppl': '10.98', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '297.6', 'tokens/total': 655360, 'tokens/trainable': 549511, 'epoch': '0.6061'} |
|
21%|████████████████████████████████████▎ | 5/24 [03:16<12:14, 38.66s/it]
25%|███████████████████████████████████████████▌ | 6/24 [03:54<11:33, 38.55s/it]
{'loss': '2.312', 'grad_norm': '6.938', 'learning_rate': '5e-06', 'ppl': '10.09', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '316.3', 'tokens/total': 786432, 'tokens/trainable': 654232, 'epoch': '0.7273'} |
|
25%|███████████████████████████████████████████▌ | 6/24 [03:54<11:33, 38.55s/it]
29%|██████████████████████████████████████████████████▊ | 7/24 [04:33<10:57, 38.67s/it]
{'loss': '2.197', 'grad_norm': '5.812', 'learning_rate': '5e-06', 'ppl': '8.994', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '369.3', 'tokens/total': 917504, 'tokens/trainable': 764296, 'epoch': '0.8485'} |
|
29%|██████████████████████████████████████████████████▊ | 7/24 [04:33<10:57, 38.67s/it]
33%|██████████████████████████████████████████████████████████ | 8/24 [05:11<10:17, 38.58s/it]
{'loss': '2.284', 'grad_norm': '5.781', 'learning_rate': '5e-06', 'ppl': '9.814', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '327.3', 'tokens/total': 1048576, 'tokens/trainable': 870990, 'epoch': '0.9697'} |
|
33%|██████████████████████████████████████████████████████████ | 8/24 [05:11<10:17, 38.58s/it]
38%|█████████████████████████████████████████████████████████████████▎ | 9/24 [05:18<07:10, 28.67s/it]
{'loss': '2.249', 'grad_norm': '9.812', 'learning_rate': '5e-06', 'ppl': '9.482', 'memory/max_active (GiB)': '62.92', 'memory/max_allocated (GiB)': '62.92', 'memory/device_reserved (GiB)': '66.81', 'tokens/train_per_sec_per_gpu': '84.22', 'tokens/total': 1069056, 'tokens/trainable': 884389, 'epoch': '1'} |
|
38%|█████████████████████████████████████████████████████████████████▎ | 9/24 [05:18<07:10, 28.67s/it][2026-05-16 18:26:39,179] [INFO] [axolotl.core.trainers.base._save:818] [PID:14603] Saving model checkpoint to ./model-output/checkpoint-9 |
| |
|
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A |
|
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:30<00:00, 30.23s/it][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:30<00:00, 30.23s/it] |
|
42%|████████████████████████████████████████████████████████████████████████ | 10/24 [07:12<12:51, 55.07s/it]
{'loss': '2.225', 'grad_norm': '4.844', 'learning_rate': '5e-06', 'ppl': '9.256', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '66.96', 'tokens/train_per_sec_per_gpu': '304.3', 'tokens/total': 1200128, 'tokens/trainable': 990168, 'epoch': '1.121'} |
|
42%|████████████████████████████████████████████████████████████████████████ | 10/24 [07:12<12:51, 55.07s/it]
46%|███████████████████████████████████████████████████████████████████████████████▎ | 11/24 [07:51<10:50, 50.02s/it]
{'loss': '2.138', 'grad_norm': '3.844', 'learning_rate': '5e-06', 'ppl': '8.483', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '346.7', 'tokens/total': 1331200, 'tokens/trainable': 1096604, 'epoch': '1.242'} |
|
46%|███████████████████████████████████████████████████████████████████████████████▎ | 11/24 [07:51<10:50, 50.02s/it]
50%|██████████████████████████████████████████████████████████████████████████████████████▌ | 12/24 [08:30<09:18, 46.55s/it]
{'loss': '2.128', 'grad_norm': '3.375', 'learning_rate': '5e-06', 'ppl': '8.399', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '356.7', 'tokens/total': 1462272, 'tokens/trainable': 1205114, 'epoch': '1.364'} |
|
50%|██████████████████████████████████████████████████████████████████████████████████████▌ | 12/24 [08:30<09:18, 46.55s/it]
54%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 13/24 [09:08<08:05, 44.16s/it]
{'loss': '2.132', 'grad_norm': '3.031', 'learning_rate': '5e-06', 'ppl': '8.428', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '365.4', 'tokens/total': 1593344, 'tokens/trainable': 1316693, 'epoch': '1.485'} |
|
54%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 13/24 [09:08<08:05, 44.16s/it]
58%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 14/24 [09:47<07:06, 42.64s/it]
{'loss': '2.098', 'grad_norm': '2.969', 'learning_rate': '5e-06', 'ppl': '8.149', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '331.6', 'tokens/total': 1724416, 'tokens/trainable': 1424652, 'epoch': '1.606'} |
|
58%|████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 14/24 [09:47<07:06, 42.64s/it]
62%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15/24 [10:26<06:12, 41.39s/it]
{'loss': '2.048', 'grad_norm': '2.922', 'learning_rate': '5e-06', 'ppl': '7.752', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '364', 'tokens/total': 1855488, 'tokens/trainable': 1533890, 'epoch': '1.727'} |
|
62%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 15/24 [10:26<06:12, 41.39s/it]
67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16/24 [11:05<05:25, 40.66s/it]
{'loss': '2.035', 'grad_norm': '2.531', 'learning_rate': '5e-06', 'ppl': '7.653', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '350.2', 'tokens/total': 1986560, 'tokens/trainable': 1645014, 'epoch': '1.848'} |
|
67%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 16/24 [11:05<05:25, 40.66s/it]
71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17/24 [11:44<04:40, 40.12s/it]
{'loss': '1.966', 'grad_norm': '2.266', 'learning_rate': '5e-06', 'ppl': '7.145', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '352.3', 'tokens/total': 2117632, 'tokens/trainable': 1754616, 'epoch': '1.97'} |
|
71%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 17/24 [11:44<04:40, 40.12s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18/24 [11:50<02:59, 29.95s/it]
{'loss': '2.165', 'grad_norm': '9.062', 'learning_rate': '5e-06', 'ppl': '8.716', 'memory/max_active (GiB)': '62.92', 'memory/max_allocated (GiB)': '62.92', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '94.88', 'tokens/total': 2138112, 'tokens/trainable': 1768778, 'epoch': '2'} |
|
75%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 18/24 [11:50<02:59, 29.95s/it][2026-05-16 18:33:10,946] [INFO] [axolotl.core.trainers.base._save:818] [PID:14603] Saving model checkpoint to ./model-output/checkpoint-18 |
| |
|
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A |
|
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:31<00:00, 31.41s/it][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:31<00:00, 31.41s/it] |
|
79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19/24 [13:46<04:39, 55.93s/it]
{'loss': '1.997', 'grad_norm': '2.172', 'learning_rate': '5e-06', 'ppl': '7.369', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '348.2', 'tokens/total': 2269184, 'tokens/trainable': 1876969, 'epoch': '2.121'} |
|
79%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 19/24 [13:47<04:39, 55.93s/it]
83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20/24 [14:25<03:22, 50.72s/it]
{'loss': '1.962', 'grad_norm': '1.953', 'learning_rate': '5e-06', 'ppl': '7.113', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '356.7', 'tokens/total': 2400256, 'tokens/trainable': 1985068, 'epoch': '2.242'} |
|
83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 20/24 [14:25<03:22, 50.72s/it]
88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21/24 [15:04<02:21, 47.12s/it]
{'loss': '1.99', 'grad_norm': '1.891', 'learning_rate': '5e-06', 'ppl': '7.317', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '354.8', 'tokens/total': 2531328, 'tokens/trainable': 2093452, 'epoch': '2.364'} |
|
88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 21/24 [15:04<02:21, 47.12s/it]
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22/24 [15:42<01:29, 44.57s/it]
{'loss': '1.998', 'grad_norm': '1.953', 'learning_rate': '5e-06', 'ppl': '7.378', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '344.1', 'tokens/total': 2662400, 'tokens/trainable': 2202303, 'epoch': '2.485'} |
|
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 22/24 [15:42<01:29, 44.57s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23/24 [16:21<00:42, 42.91s/it]
{'loss': '1.957', 'grad_norm': '1.922', 'learning_rate': '5e-06', 'ppl': '7.076', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '341.9', 'tokens/total': 2793472, 'tokens/trainable': 2308487, 'epoch': '2.606'} |
|
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 23/24 [16:21<00:42, 42.91s/it]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [17:00<00:00, 41.56s/it]
{'loss': '1.918', 'grad_norm': '1.797', 'learning_rate': '5e-06', 'ppl': '6.805', 'memory/max_active (GiB)': '65.97', 'memory/max_allocated (GiB)': '65.97', 'memory/device_reserved (GiB)': '67.04', 'tokens/train_per_sec_per_gpu': '358.8', 'tokens/total': 2924544, 'tokens/trainable': 2417420, 'epoch': '2.727'} |
|
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [17:00<00:00, 41.56s/it][2026-05-16 18:38:20,770] [INFO] [axolotl.core.trainers.base._save:818] [PID:14603] Saving model checkpoint to ./model-output/checkpoint-24 |
| |
|
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s][A |
|
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:33<00:00, 33.18s/it][A
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:33<00:00, 33.18s/it] |
|
{'train_runtime': '1112', 'train_samples_per_second': '0.691', 'train_steps_per_second': '0.022', 'train_loss': '2.199', 'memory/max_active (GiB)': '45.02', 'memory/max_allocated (GiB)': '45.02', 'memory/device_reserved (GiB)': '67.04', 'epoch': '2.727', 'tokens/train_per_sec_per_gpu': '0'} |
|
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [18:18<00:00, 41.56s/it]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 24/24 [18:18<00:00, 45.79s/it] |
| [2026-05-16 18:39:39,356] [INFO] [axolotl.train.save_trained_model:267] [PID:14603] Training completed! Saving trained model to ./model-output. |
|
Writing model shards: 0%| | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:28<00:00, 28.84s/it]
Writing model shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:28<00:00, 28.85s/it] |
| [2026-05-16 18:40:08,236] [INFO] [axolotl.train.save_trained_model:388] [PID:14603] Model successfully saved to ./model-output |
| |
