debug.log

[2026-01-27 12:10:11,774] [DEBUG] [axolotl.utils.config.resolve_dtype:66] [PID:168] bf16 support detected, enabling for this configuration.
[2026-01-27 12:10:11,777] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:168] baseline 0.000GB ()
[2026-01-27 12:10:11,777] [INFO] [axolotl.cli.config.load_cfg:259] [PID:168] config:
{
  "activation_offloading": false,
  "adapter": "lora",
  "axolotl_config_path": "/workspace/axolotl/configs/1.yml",
  "base_model": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
  "base_model_config": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
  "batch_size": 8,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_90",
    "fp8": true,
    "n_gpu": 1,
    "n_node": 1
  },
  "chat_template": "llama3",
  "context_parallel_size": 1,
  "dataloader_num_workers": 1,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_num_proc": 32,
  "datasets": [
    {
      "data_files": [
        "1_train_data.json"
      ],
      "ds_type": "json",
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "/workspace/axolotl/data",
      "split": "train",
      "trust_remote_code": false
    }
  ],
  "ddp": false,
  "device": "cuda:0",
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "env_capabilities": {
    "torch_version": "2.8.0"
  },
  "eval_batch_size": 8,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 128,
  "eval_strategy": "no",
  "eval_table_size": 0,
  "experimental_skip_move_to_device": true,
  "flash_attention": false,
  "fp16": false,
  "gradient_accumulation_steps": 1,
  "gradient_checkpointing": false,
  "group_by_length": false,
  "include_tkps": true,
  "is_falcon_derived_model": false,
  "is_llama_derived_model": false,
  "is_mistral_derived_model": false,
  "learning_rate": 7e-06,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 1,
  "lora_alpha": 32,
  "lora_dropout": 0.0,
  "lora_r": 64,
  "lora_target_linear": true,
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "max_grad_norm": 1.0,
  "max_steps": 100000,
  "mean_resizing_embeddings": false,
  "micro_batch_size": 8,
  "mlflow_experiment_name": "/workspace/axolotl/data/1_train_data.json",
  "model_config_type": "qwen2",
  "num_epochs": 1.0,
  "optimizer": "adamw_bnb_8bit",
  "otel_metrics_host": "localhost",
  "otel_metrics_port": 8000,
  "output_dir": "/app/checkpoints/1/environment_test_affine",
  "pad_to_sequence_len": true,
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qlora_sharded_model_loading": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "rl": "grpo",
  "sample_packing": false,
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": false,
  "save_safetensors": true,
  "save_steps": 10,
  "save_total_limit": 1,
  "sequence_len": 24000,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "special_tokens": {
    "bos_token": "<|im_end|>"
  },
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tf32": false,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
  "tokenizer_save_jinja_files": true,
  "tokenizer_type": "AutoTokenizer",
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "beta": 0.001,
    "log_completions": false,
    "mask_truncated_completions": false,
    "max_completion_length": 512,
    "num_generations": 8,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "reward_funcs": [
      "affine_game.rollout_reward_func"
    ],
    "reward_weights": [
      1.0
    ],
    "rollout_func": "affine_game.rollout_first_prompt_and_completion",
    "scale_rewards": true,
    "sync_ref_model": false,
    "temperature": 0.7,
    "use_vllm": true,
    "vllm_enable_sleep_mode": false,
    "vllm_mode": "colocate",
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "trust_remote_code": true,
  "type_of_model": "AutoModelForCausalLM",
  "use_mlflow": true,
  "use_otel_metrics": false,
  "use_ray": false,
  "use_wandb": true,
  "val_set_size": 0.0,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "enable_prefix_caching": false,
    "gpu_memory_utilization": 0.15,
    "host": "0.0.0.0",
    "max_model_len": 24000,
    "port": 8000,
    "tensor_parallel_size": 1
  },
  "wandb_mode": "online",
  "wandb_name": "1_environment_test_affine",
  "wandb_project": "Affine-GAME-Tests",
  "warmup_steps": 20,
  "weight_decay": 0.0,
  "world_size": 1
}
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:168] EOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:168] BOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:168] PAD: 151643 / <|endoftext|>
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:168] UNK: None / None
[2026-01-27 12:10:12,210] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:168] Unable to find prepared dataset in last_run_prepared/ba0ae834220c702ae7aefbdbfde66c85

Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 1000 examples [00:00, 123307.48 examples/s]
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:168] EOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:168] BOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:168] PAD: 151643 / <|endoftext|>
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:168] UNK: None / None

Dropping Long Sequences (num_proc=32):   0%|          | 0/1000 [00:00<?, ? examples/s]
Dropping Long Sequences (num_proc=32):   3%|▎         | 32/1000 [00:00<00:27, 35.34 examples/s]
Dropping Long Sequences (num_proc=32):  10%|▉         | 96/1000 [00:01<00:07, 117.01 examples/s]
Dropping Long Sequences (num_proc=32):  16%|█▌        | 160/1000 [00:01<00:04, 186.99 examples/s]
Dropping Long Sequences (num_proc=32):  22%|██▏       | 224/1000 [00:01<00:03, 227.92 examples/s]
Dropping Long Sequences (num_proc=32):  29%|██▊       | 287/1000 [00:01<00:02, 277.18 examples/s]
Dropping Long Sequences (num_proc=32):  35%|███▍      | 349/1000 [00:01<00:02, 312.62 examples/s]
Dropping Long Sequences (num_proc=32):  41%|████      | 411/1000 [00:01<00:01, 333.29 examples/s]
Dropping Long Sequences (num_proc=32):  47%|████▋     | 473/1000 [00:01<00:01, 338.99 examples/s]
Dropping Long Sequences (num_proc=32):  54%|█████▎    | 535/1000 [00:02<00:01, 347.40 examples/s]
Dropping Long Sequences (num_proc=32):  60%|█████▉    | 597/1000 [00:02<00:01, 350.00 examples/s]
Dropping Long Sequences (num_proc=32):  66%|██████▌   | 659/1000 [00:02<00:00, 362.42 examples/s]
Dropping Long Sequences (num_proc=32):  72%|███████▏  | 721/1000 [00:02<00:00, 378.68 examples/s]
Dropping Long Sequences (num_proc=32):  78%|███████▊  | 783/1000 [00:02<00:00, 359.18 examples/s]
Dropping Long Sequences (num_proc=32):  84%|████████▍ | 845/1000 [00:02<00:00, 384.00 examples/s]
Dropping Long Sequences (num_proc=32):  91%|█████████ | 907/1000 [00:03<00:00, 429.74 examples/s]
Dropping Long Sequences (num_proc=32):  97%|█████████▋| 969/1000 [00:03<00:00, 433.14 examples/s]
Dropping Long Sequences (num_proc=32): 100%|██████████| 1000/1000 [00:03<00:00, 278.63 examples/s]

Saving the dataset (0/3 shards):   0%|          | 0/1000 [00:00<?, ? examples/s]
Saving the dataset (1/3 shards):  33%|███▎      | 334/1000 [00:00<00:00, 3720.68 examples/s]
Saving the dataset (2/3 shards):  67%|██████▋   | 667/1000 [00:00<00:00, 7231.20 examples/s]
Saving the dataset (3/3 shards): 100%|██████████| 1000/1000 [00:00<00:00, 10699.73 examples/s]
Saving the dataset (3/3 shards): 100%|██████████| 1000/1000 [00:00<00:00, 5825.09 examples/s] 
[2026-01-27 12:10:16,798] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:168] loading tokenizer... /cache/models/Qwen--Qwen2.5-3B-Instruct
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:168] EOS: 151645 / <|im_end|>
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:168] BOS: 151645 / <|im_end|>
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:168] PAD: 151643 / <|endoftext|>
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:168] UNK: None / None
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.train.setup_model_and_tokenizer:82] [PID:168] Loading model
[2026-01-27 12:10:16,986] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:168] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-01-27 12:10:16,987] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:168] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation

Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00, 86.03it/s]
[2026-01-27 12:10:17,609] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:347] [PID:168] Converting modules to torch.bfloat16
[2026-01-27 12:10:18,005] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:168] Memory usage after model load 0.000GB ()
[2026-01-27 12:10:18,005] [INFO] [axolotl.loaders.adapter.load_lora:81] [PID:168] found linear modules: ['down_proj', 'gate_proj', 'k_proj', 'o_proj', 'q_proj', 'up_proj', 'v_proj']
trainable params: 119,734,272 || all params: 3,205,672,960 || trainable%: 3.7351
[2026-01-27 12:10:18,865] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:168] after adapters 0.000GB ()
[2026-01-27 12:10:19,505] [DEBUG] [axolotl.train.setup_reference_model:126] [PID:168] Passing model_ref: None to RL trainer
[2026-01-27 12:10:25,633] [WARNING] [py.warnings._showwarnmsg:110] [PID:168] /workspace/axolotl/src/axolotl/core/trainers/mixins/optimizer.py:209: UserWarning: You are importing from 'rollout_func', which is an experimental feature. This API may change or be removed at any time without prior notice. Silence this warning by setting environment variable TRL_EXPERIMENTAL_SILENCE=1.
  super().__init__(*args, **kwargs)


Loading safetensors checkpoint shards:   0% Completed | 0/2 [00:00<?, ?it/s]

Loading safetensors checkpoint shards:  50% Completed | 1/2 [00:00<00:00,  4.17it/s]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:00<00:00,  2.61it/s]

Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:00<00:00,  2.77it/s]


Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):   0%|          | 0/5 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE):  80%|████████  | 4/5 [00:00<00:00, 34.65it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 5/5 [00:00<00:00, 32.94it/s]
[2026-01-27 12:11:01,350] [INFO] [axolotl.train.save_initial_configs:413] [PID:168] Pre-saving adapter config to /app/checkpoints/1/environment_test_affine...
[2026-01-27 12:11:01,350] [INFO] [axolotl.train.save_initial_configs:417] [PID:168] Pre-saving tokenizer to /app/checkpoints/1/environment_test_affine...
[2026-01-27 12:11:01,463] [INFO] [axolotl.train.save_initial_configs:422] [PID:168] Pre-saving model config to /app/checkpoints/1/environment_test_affine...
[2026-01-27 12:11:01,466] [INFO] [axolotl.train.execute_training:212] [PID:168] Starting trainer...
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
wandb: Currently logged in as: bkbvol (bkbvol-bittensor) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: setting up run 34sye9hd
wandb: Tracking run with wandb version 0.24.0
wandb: Run data is saved locally in /workspace/axolotl/wandb/run-20260127_121102-34sye9hd
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run 1_environment_test_affine
wandb: ⭐️ View project at https://wandb.ai/bkbvol-bittensor/Affine-GAME-Tests
wandb: 🚀 View run at https://wandb.ai/bkbvol-bittensor/Affine-GAME-Tests/runs/34sye9hd
wandb: Detected [huggingface_hub.inference, openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
    fire.Fire(do_cli)
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
    component_trace = _Fire(component, args, parsed_flag_args, context, name)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
    component, remaining_args = _CallAndUpdateTrace(
                                ^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
    component = fn(*varargs, **kwargs)
                ^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
    return do_train(parsed_cfg, parsed_cli_args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/cli/train.py", line 45, in do_train
    model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta)
                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/workspace/axolotl/src/axolotl/train.py", line 598, in train
    execute_training(cfg, trainer, resume_from_checkpoint)
  File "/workspace/axolotl/src/axolotl/train.py", line 213, in execute_training
    trainer.train(resume_from_checkpoint=resume_from_checkpoint)
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2325, in train
    return inner_training_loop(
           ^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2573, in _inner_training_loop
    self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 506, in on_train_begin
    return self.call_event("on_train_begin", args, state, control)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 556, in call_event
    result = getattr(callback, event)(
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1489, in on_train_begin
    self.setup(args, state, model)
  File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1430, in setup
    if not self._ml_flow.is_tracking_uri_set():
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: module 'mlflow' has no attribute 'is_tracking_uri_set'
[rank0]: Traceback (most recent call last):
[rank0]:   File "<frozen runpy>", line 198, in _run_module_as_main
[rank0]:   File "<frozen runpy>", line 88, in _run_code
[rank0]:   File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
[rank0]:     fire.Fire(do_cli)
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
[rank0]:     component_trace = _Fire(component, args, parsed_flag_args, context, name)
[rank0]:                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
[rank0]:     component, remaining_args = _CallAndUpdateTrace(
[rank0]:                                 ^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
[rank0]:     component = fn(*varargs, **kwargs)
[rank0]:                 ^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
[rank0]:     return do_train(parsed_cfg, parsed_cli_args)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/workspace/axolotl/src/axolotl/cli/train.py", line 45, in do_train
[rank0]:     model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta)
[rank0]:                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/workspace/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper
[rank0]:     return func(*args, **kwargs)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/workspace/axolotl/src/axolotl/train.py", line 598, in train
[rank0]:     execute_training(cfg, trainer, resume_from_checkpoint)
[rank0]:   File "/workspace/axolotl/src/axolotl/train.py", line 213, in execute_training
[rank0]:     trainer.train(resume_from_checkpoint=resume_from_checkpoint)
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2325, in train
[rank0]:     return inner_training_loop(
[rank0]:            ^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2573, in _inner_training_loop
[rank0]:     self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
[rank0]:                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 506, in on_train_begin
[rank0]:     return self.call_event("on_train_begin", args, state, control)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 556, in call_event
[rank0]:     result = getattr(callback, event)(
[rank0]:              ^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1489, in on_train_begin
[rank0]:     self.setup(args, state, model)
[rank0]:   File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1430, in setup
[rank0]:     if not self._ml_flow.is_tracking_uri_set():
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: AttributeError: module 'mlflow' has no attribute 'is_tracking_uri_set'