[2026-01-10 17:09:55,819] [DEBUG] [axolotl.utils.config.resolve_dtype:66] [PID:621358] bf16 support detected, enabling for this configuration. [2026-01-10 17:09:56,050] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:621358] baseline 0.000GB () [2026-01-10 17:09:56,051] [INFO] [axolotl.cli.config.load_cfg:263] [PID:621358] config: { "activation_offloading": false, "adapter": "qlora", "axolotl_config_path": "olmo-stage1.yaml", "base_model": "allenai/Olmo-3.1-32B-Instruct", "base_model_config": "allenai/Olmo-3.1-32B-Instruct", "batch_size": 2, "bf16": true, "capabilities": { "bf16": true, "compute_capability": "sm_86", "fp8": false, "n_gpu": 1, "n_node": 1 }, "context_parallel_size": 1, "cut_cross_entropy": true, "dataloader_num_workers": 1, "dataloader_pin_memory": true, "dataloader_prefetch_factor": 256, "dataset_num_proc": 24, "dataset_prepared_path": "last_run_prepared", "datasets": [ { "message_property_mappings": { "content": "content", "role": "role" }, "path": "../marvin_philip_k_dick.json", "trust_remote_code": false, "type": "completion" }, { "message_property_mappings": { "content": "content", "role": "role" }, "path": "../marvin_thomas_ligotti.json", "trust_remote_code": false, "type": "completion" }, { "message_property_mappings": { "content": "content", "role": "role" }, "path": "../marvin_greg_egan.json", "trust_remote_code": false, "type": "completion" }, { "message_property_mappings": { "content": "content", "role": "role" }, "path": "../erotica_quality_trimmed.json", "trust_remote_code": false, "type": "completion" } ], "ddp": false, "device": "cuda:0", "device_map": "auto", "dion_rank_fraction": 1.0, "dion_rank_multiple_of": 1, "env_capabilities": { "torch_version": "2.9.1" }, "eval_batch_size": 1, "eval_causal_lm_metrics": [ "sacrebleu", "comet", "ter", "chrf" ], "eval_max_new_tokens": 128, "eval_sample_packing": true, "eval_steps": 0.2, "eval_table_size": 0, "evals_per_epoch": 5, "experimental_skip_move_to_device": true, "flash_attention": false, "fp16": false, "gc_steps": 10, "gradient_accumulation_steps": 2, "gradient_checkpointing": false, "group_by_length": false, "include_tkps": true, "learning_rate": 1e-05, "liger_glu_activation": true, "liger_layer_norm": true, "liger_rms_norm": true, "liger_rope": true, "lisa_layers_attribute": "model.layers", "load_best_model_at_end": false, "load_in_4bit": false, "load_in_8bit": false, "local_rank": 0, "logging_steps": 1, "lora_alpha": 8, "lora_dropout": 0.05, "lora_r": 32, "lora_target_linear": true, "loraplus_lr_embedding": 1e-06, "lr_scheduler": "cosine", "max_grad_norm": 1.0, "mean_resizing_embeddings": false, "merge_lora": true, "micro_batch_size": 1, "model_config_type": "olmo3", "num_epochs": 1.0, "optimizer": "adamw_torch_fused", "otel_metrics_host": "localhost", "otel_metrics_port": 8000, "output_dir": "ckpts-olmo-qlora/marvin", "pad_to_sequence_len": true, "peft_use_rslora": true, "plugins": [ "axolotl.integrations.liger.LigerPlugin", "axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin" ], "pretrain_multipack_attn": true, "profiler_steps_start": 0, "qlora_sharded_model_loading": false, "ray_num_workers": 1, "resources_per_worker": { "GPU": 1 }, "sample_packing": true, "sample_packing_bin_size": 200, "sample_packing_group_size": 100000, "save_only_model": false, "save_safetensors": true, "save_total_limit": 2, "saves_per_epoch": 1, "seed": 69, "sequence_len": 16384, "shuffle_before_merging_datasets": false, "shuffle_merged_datasets": true, "skip_prepare_dataset": false, "streaming_multipack_buffer_size": 10000, "strict": false, "tensor_parallel_size": 1, "tiled_mlp_use_original_mlp": true, "tokenizer_config": "allenai/Olmo-3.1-32B-Instruct", "tokenizer_save_jinja_files": true, "torch_dtype": "torch.bfloat16", "train_on_inputs": false, "trl": { "log_completions": false, "mask_truncated_completions": false, "ref_model_mixup_alpha": 0.9, "ref_model_sync_steps": 64, "scale_rewards": true, "sync_ref_model": false, "use_vllm": false, "vllm_server_host": "0.0.0.0", "vllm_server_port": 8000 }, "use_otel_metrics": false, "use_ray": false, "use_wandb": true, "val_set_size": 0.025, "vllm": { "device": "auto", "dtype": "auto", "gpu_memory_utilization": 0.9, "host": "0.0.0.0", "port": 8000 }, "wandb_name": "marvin-qlora", "wandb_project": "Olmo3", "warmup_ratio": 0.025, "weight_decay": 0.01, "world_size": 1 } [2026-01-10 17:09:56,051] [INFO] [axolotl.cli.utils.load.load_model_and_tokenizer:40] [PID:621358] loading tokenizer... allenai/Olmo-3.1-32B-Instruct [2026-01-10 17:09:56,690] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:280] [PID:621358] EOS: 100257 / <|endoftext|> [2026-01-10 17:09:56,690] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:281] [PID:621358] BOS: 100257 / <|endoftext|> [2026-01-10 17:09:56,690] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:282] [PID:621358] PAD: 100277 / <|pad|> [2026-01-10 17:09:56,690] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:283] [PID:621358] UNK: 100257 / <|endoftext|> [2026-01-10 17:09:56,691] [INFO] [axolotl.cli.utils.load.load_model_and_tokenizer:43] [PID:621358] loading model... [2026-01-10 17:09:56,852] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:621358] Patched Trainer.evaluation_loop with nanmean loss calculation [2026-01-10 17:09:56,853] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:621358] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation [2026-01-10 17:09:56,854] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:301] [PID:621358] Applying multipack dataloader patch for sample packing... [2026-01-10 17:09:56,932] [INFO] [axolotl.integrations.liger.plugin.pre_model_load:98] [PID:621358] Applying LIGER to olmo3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': None, 'rms_norm': True, 'swiglu': True} [2026-01-10 17:09:56,997] [INFO] [axolotl.integrations.cut_cross_entropy.pre_model_load:94] [PID:621358] Applying Cut Cross Entropy to model type: olmo3 Loading checkpoint shards: 0%| | 0/14 [00:00 sys.exit(main()) ^^^^^^ File "/home/aibox/training/axolotl/src/axolotl/cli/main.py", line 347, in main cli() File "/home/aibox/axo/lib/python3.12/site-packages/click/core.py", line 1161, in __call__ return self.main(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/axo/lib/python3.12/site-packages/click/core.py", line 1082, in main rv = self.invoke(ctx) ^^^^^^^^^^^^^^^^ File "/home/aibox/axo/lib/python3.12/site-packages/click/core.py", line 1697, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/axo/lib/python3.12/site-packages/click/core.py", line 1443, in invoke return ctx.invoke(self.callback, **ctx.params) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/axo/lib/python3.12/site-packages/click/core.py", line 788, in invoke return __callback(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/training/axolotl/src/axolotl/cli/utils/args.py", line 47, in wrapper return func(*args, **filtered_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/training/axolotl/src/axolotl/cli/main.py", line 293, in merge_lora do_cli(config=config, **kwargs) File "/home/aibox/training/axolotl/src/axolotl/cli/merge_lora.py", line 91, in do_cli do_merge_lora(cfg=parsed_cfg) File "/home/aibox/training/axolotl/src/axolotl/telemetry/errors.py", line 127, in wrapper return func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/training/axolotl/src/axolotl/cli/merge_lora.py", line 26, in do_merge_lora model, tokenizer, processor = load_model_and_tokenizer(cfg=cfg) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/training/axolotl/src/axolotl/cli/utils/load.py", line 45, in load_model_and_tokenizer model, _ = model_loader.load() ^^^^^^^^^^^^^^^^^^^ File "/home/aibox/training/axolotl/src/axolotl/telemetry/errors.py", line 127, in wrapper return func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/training/axolotl/src/axolotl/loaders/model.py", line 180, in load self._apply_post_model_load_setup() File "/home/aibox/training/axolotl/src/axolotl/loaders/model.py", line 232, in _apply_post_model_load_setup self._configure_embedding_dtypes() File "/home/aibox/training/axolotl/src/axolotl/loaders/model.py", line 308, in _configure_embedding_dtypes self._convert_embedding_modules_dtype( File "/home/aibox/training/axolotl/src/axolotl/loaders/model.py", line 877, in _convert_embedding_modules_dtype module.to(**dest) File "/home/aibox/axo/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1371, in to return self._apply(convert) ^^^^^^^^^^^^^^^^^^^^ File "/home/aibox/axo/lib/python3.12/site-packages/torch/nn/modules/module.py", line 957, in _apply param_applied = fn(param) ^^^^^^^^^ File "/home/aibox/axo/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1357, in convert return t.to( ^^^^^ torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 1.91 GiB. GPU 0 has a total capacity of 23.57 GiB of which 1.46 GiB is free. Including non-PyTorch memory, this process has 22.10 GiB memory in use. Of the allocated memory 21.85 GiB is allocated by PyTorch, and 10.00 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)