Gege24's picture
Upload task output 1
dd40b7a verified
[2026-01-27 12:10:11,774] [DEBUG] [axolotl.utils.config.resolve_dtype:66] [PID:168] bf16 support detected, enabling for this configuration.
[2026-01-27 12:10:11,777] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:168] baseline 0.000GB ()
[2026-01-27 12:10:11,777] [INFO] [axolotl.cli.config.load_cfg:259] [PID:168] config:
{
"activation_offloading": false,
"adapter": "lora",
"axolotl_config_path": "/workspace/axolotl/configs/1.yml",
"base_model": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
"base_model_config": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
"batch_size": 8,
"bf16": true,
"capabilities": {
"bf16": true,
"compute_capability": "sm_90",
"fp8": true,
"n_gpu": 1,
"n_node": 1
},
"chat_template": "llama3",
"context_parallel_size": 1,
"dataloader_num_workers": 1,
"dataloader_pin_memory": true,
"dataloader_prefetch_factor": 256,
"dataset_num_proc": 32,
"datasets": [
{
"data_files": [
"1_train_data.json"
],
"ds_type": "json",
"message_property_mappings": {
"content": "content",
"role": "role"
},
"path": "/workspace/axolotl/data",
"split": "train",
"trust_remote_code": false
}
],
"ddp": false,
"device": "cuda:0",
"dion_rank_fraction": 1.0,
"dion_rank_multiple_of": 1,
"env_capabilities": {
"torch_version": "2.8.0"
},
"eval_batch_size": 8,
"eval_causal_lm_metrics": [
"sacrebleu",
"comet",
"ter",
"chrf"
],
"eval_max_new_tokens": 128,
"eval_strategy": "no",
"eval_table_size": 0,
"experimental_skip_move_to_device": true,
"flash_attention": false,
"fp16": false,
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"group_by_length": false,
"include_tkps": true,
"is_falcon_derived_model": false,
"is_llama_derived_model": false,
"is_mistral_derived_model": false,
"learning_rate": 7e-06,
"lisa_layers_attribute": "model.layers",
"load_best_model_at_end": false,
"load_in_4bit": false,
"load_in_8bit": false,
"local_rank": 0,
"logging_steps": 1,
"lora_alpha": 32,
"lora_dropout": 0.0,
"lora_r": 64,
"lora_target_linear": true,
"loraplus_lr_embedding": 1e-06,
"lr_scheduler": "cosine",
"max_grad_norm": 1.0,
"max_steps": 100000,
"mean_resizing_embeddings": false,
"micro_batch_size": 8,
"mlflow_experiment_name": "/workspace/axolotl/data/1_train_data.json",
"model_config_type": "qwen2",
"num_epochs": 1.0,
"optimizer": "adamw_bnb_8bit",
"otel_metrics_host": "localhost",
"otel_metrics_port": 8000,
"output_dir": "/app/checkpoints/1/environment_test_affine",
"pad_to_sequence_len": true,
"pretrain_multipack_attn": true,
"profiler_steps_start": 0,
"qlora_sharded_model_loading": false,
"ray_num_workers": 1,
"resources_per_worker": {
"GPU": 1
},
"rl": "grpo",
"sample_packing": false,
"sample_packing_bin_size": 200,
"sample_packing_group_size": 100000,
"save_only_model": false,
"save_safetensors": true,
"save_steps": 10,
"save_total_limit": 1,
"sequence_len": 24000,
"shuffle_before_merging_datasets": false,
"shuffle_merged_datasets": true,
"skip_prepare_dataset": false,
"special_tokens": {
"bos_token": "<|im_end|>"
},
"streaming_multipack_buffer_size": 10000,
"strict": false,
"tensor_parallel_size": 1,
"tf32": false,
"tiled_mlp_use_original_mlp": true,
"tokenizer_config": "/cache/models/Qwen--Qwen2.5-3B-Instruct",
"tokenizer_save_jinja_files": true,
"tokenizer_type": "AutoTokenizer",
"torch_dtype": "torch.bfloat16",
"train_on_inputs": false,
"trl": {
"beta": 0.001,
"log_completions": false,
"mask_truncated_completions": false,
"max_completion_length": 512,
"num_generations": 8,
"ref_model_mixup_alpha": 0.9,
"ref_model_sync_steps": 64,
"reward_funcs": [
"affine_game.rollout_reward_func"
],
"reward_weights": [
1.0
],
"rollout_func": "affine_game.rollout_first_prompt_and_completion",
"scale_rewards": true,
"sync_ref_model": false,
"temperature": 0.7,
"use_vllm": true,
"vllm_enable_sleep_mode": false,
"vllm_mode": "colocate",
"vllm_server_host": "0.0.0.0",
"vllm_server_port": 8000
},
"trust_remote_code": true,
"type_of_model": "AutoModelForCausalLM",
"use_mlflow": true,
"use_otel_metrics": false,
"use_ray": false,
"use_wandb": true,
"val_set_size": 0.0,
"vllm": {
"device": "auto",
"dtype": "auto",
"enable_prefix_caching": false,
"gpu_memory_utilization": 0.15,
"host": "0.0.0.0",
"max_model_len": 24000,
"port": 8000,
"tensor_parallel_size": 1
},
"wandb_mode": "online",
"wandb_name": "1_environment_test_affine",
"wandb_project": "Affine-GAME-Tests",
"warmup_steps": 20,
"weight_decay": 0.0,
"world_size": 1
}
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:168] EOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:168] BOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:168] PAD: 151643 / <|endoftext|>
[2026-01-27 12:10:12,210] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:168] UNK: None / None
[2026-01-27 12:10:12,210] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:481] [PID:168] Unable to find prepared dataset in last_run_prepared/ba0ae834220c702ae7aefbdbfde66c85
Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 1000 examples [00:00, 123307.48 examples/s]
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:168] EOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:168] BOS: 151645 / <|im_end|>
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:168] PAD: 151643 / <|endoftext|>
[2026-01-27 12:10:12,835] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:168] UNK: None / None
Dropping Long Sequences (num_proc=32): 0%| | 0/1000 [00:00<?, ? examples/s] Dropping Long Sequences (num_proc=32): 3%|β–Ž | 32/1000 [00:00<00:27, 35.34 examples/s] Dropping Long Sequences (num_proc=32): 10%|β–‰ | 96/1000 [00:01<00:07, 117.01 examples/s] Dropping Long Sequences (num_proc=32): 16%|β–ˆβ–Œ | 160/1000 [00:01<00:04, 186.99 examples/s] Dropping Long Sequences (num_proc=32): 22%|β–ˆβ–ˆβ– | 224/1000 [00:01<00:03, 227.92 examples/s] Dropping Long Sequences (num_proc=32): 29%|β–ˆβ–ˆβ–Š | 287/1000 [00:01<00:02, 277.18 examples/s] Dropping Long Sequences (num_proc=32): 35%|β–ˆβ–ˆβ–ˆβ– | 349/1000 [00:01<00:02, 312.62 examples/s] Dropping Long Sequences (num_proc=32): 41%|β–ˆβ–ˆβ–ˆβ–ˆ | 411/1000 [00:01<00:01, 333.29 examples/s] Dropping Long Sequences (num_proc=32): 47%|β–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 473/1000 [00:01<00:01, 338.99 examples/s] Dropping Long Sequences (num_proc=32): 54%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Ž | 535/1000 [00:02<00:01, 347.40 examples/s] Dropping Long Sequences (num_proc=32): 60%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‰ | 597/1000 [00:02<00:01, 350.00 examples/s] Dropping Long Sequences (num_proc=32): 66%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Œ | 659/1000 [00:02<00:00, 362.42 examples/s] Dropping Long Sequences (num_proc=32): 72%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 721/1000 [00:02<00:00, 378.68 examples/s] Dropping Long Sequences (num_proc=32): 78%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 783/1000 [00:02<00:00, 359.18 examples/s] Dropping Long Sequences (num_proc=32): 84%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ– | 845/1000 [00:02<00:00, 384.00 examples/s] Dropping Long Sequences (num_proc=32): 91%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 907/1000 [00:03<00:00, 429.74 examples/s] Dropping Long Sequences (num_proc=32): 97%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹| 969/1000 [00:03<00:00, 433.14 examples/s] Dropping Long Sequences (num_proc=32): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1000/1000 [00:03<00:00, 278.63 examples/s]
Saving the dataset (0/3 shards): 0%| | 0/1000 [00:00<?, ? examples/s] Saving the dataset (1/3 shards): 33%|β–ˆβ–ˆβ–ˆβ–Ž | 334/1000 [00:00<00:00, 3720.68 examples/s] Saving the dataset (2/3 shards): 67%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 667/1000 [00:00<00:00, 7231.20 examples/s] Saving the dataset (3/3 shards): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1000/1000 [00:00<00:00, 10699.73 examples/s] Saving the dataset (3/3 shards): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1000/1000 [00:00<00:00, 5825.09 examples/s]
[2026-01-27 12:10:16,798] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:168] loading tokenizer... /cache/models/Qwen--Qwen2.5-3B-Instruct
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:168] EOS: 151645 / <|im_end|>
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:168] BOS: 151645 / <|im_end|>
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:168] PAD: 151643 / <|endoftext|>
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:168] UNK: None / None
[2026-01-27 12:10:16,974] [DEBUG] [axolotl.train.setup_model_and_tokenizer:82] [PID:168] Loading model
[2026-01-27 12:10:16,986] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:168] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-01-27 12:10:16,987] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:168] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s] Loading checkpoint shards: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 2/2 [00:00<00:00, 86.03it/s]
[2026-01-27 12:10:17,609] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:347] [PID:168] Converting modules to torch.bfloat16
[2026-01-27 12:10:18,005] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:168] Memory usage after model load 0.000GB ()
[2026-01-27 12:10:18,005] [INFO] [axolotl.loaders.adapter.load_lora:81] [PID:168] found linear modules: ['down_proj', 'gate_proj', 'k_proj', 'o_proj', 'q_proj', 'up_proj', 'v_proj']
trainable params: 119,734,272 || all params: 3,205,672,960 || trainable%: 3.7351
[2026-01-27 12:10:18,865] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:168] after adapters 0.000GB ()
[2026-01-27 12:10:19,505] [DEBUG] [axolotl.train.setup_reference_model:126] [PID:168] Passing model_ref: None to RL trainer
[2026-01-27 12:10:25,633] [WARNING] [py.warnings._showwarnmsg:110] [PID:168] /workspace/axolotl/src/axolotl/core/trainers/mixins/optimizer.py:209: UserWarning: You are importing from 'rollout_func', which is an experimental feature. This API may change or be removed at any time without prior notice. Silence this warning by setting environment variable TRL_EXPERIMENTAL_SILENCE=1.
super().__init__(*args, **kwargs)
Loading safetensors checkpoint shards: 0% Completed | 0/2 [00:00<?, ?it/s]
Loading safetensors checkpoint shards: 50% Completed | 1/2 [00:00<00:00, 4.17it/s]
Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:00<00:00, 2.61it/s]
Loading safetensors checkpoint shards: 100% Completed | 2/2 [00:00<00:00, 2.77it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/5 [00:00<?, ?it/s] Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ | 4/5 [00:00<00:00, 34.65it/s] Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 5/5 [00:00<00:00, 32.94it/s]
[2026-01-27 12:11:01,350] [INFO] [axolotl.train.save_initial_configs:413] [PID:168] Pre-saving adapter config to /app/checkpoints/1/environment_test_affine...
[2026-01-27 12:11:01,350] [INFO] [axolotl.train.save_initial_configs:417] [PID:168] Pre-saving tokenizer to /app/checkpoints/1/environment_test_affine...
[2026-01-27 12:11:01,463] [INFO] [axolotl.train.save_initial_configs:422] [PID:168] Pre-saving model config to /app/checkpoints/1/environment_test_affine...
[2026-01-27 12:11:01,466] [INFO] [axolotl.train.execute_training:212] [PID:168] Starting trainer...
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
wandb: Currently logged in as: bkbvol (bkbvol-bittensor) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: setting up run 34sye9hd
wandb: Tracking run with wandb version 0.24.0
wandb: Run data is saved locally in /workspace/axolotl/wandb/run-20260127_121102-34sye9hd
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run 1_environment_test_affine
wandb: ⭐️ View project at https://wandb.ai/bkbvol-bittensor/Affine-GAME-Tests
wandb: πŸš€ View run at https://wandb.ai/bkbvol-bittensor/Affine-GAME-Tests/runs/34sye9hd
wandb: Detected [huggingface_hub.inference, openai] in use.
wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
fire.Fire(do_cli)
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
component, remaining_args = _CallAndUpdateTrace(
^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
return do_train(parsed_cfg, parsed_cli_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/cli/train.py", line 45, in do_train
model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/workspace/axolotl/src/axolotl/train.py", line 598, in train
execute_training(cfg, trainer, resume_from_checkpoint)
File "/workspace/axolotl/src/axolotl/train.py", line 213, in execute_training
trainer.train(resume_from_checkpoint=resume_from_checkpoint)
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2325, in train
return inner_training_loop(
^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2573, in _inner_training_loop
self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 506, in on_train_begin
return self.call_event("on_train_begin", args, state, control)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 556, in call_event
result = getattr(callback, event)(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1489, in on_train_begin
self.setup(args, state, model)
File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1430, in setup
if not self._ml_flow.is_tracking_uri_set():
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: module 'mlflow' has no attribute 'is_tracking_uri_set'
[rank0]: Traceback (most recent call last):
[rank0]: File "<frozen runpy>", line 198, in _run_module_as_main
[rank0]: File "<frozen runpy>", line 88, in _run_code
[rank0]: File "/workspace/axolotl/src/axolotl/cli/train.py", line 121, in <module>
[rank0]: fire.Fire(do_cli)
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 135, in Fire
[rank0]: component_trace = _Fire(component, args, parsed_flag_args, context, name)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 468, in _Fire
[rank0]: component, remaining_args = _CallAndUpdateTrace(
[rank0]: ^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
[rank0]: component = fn(*varargs, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/axolotl/src/axolotl/cli/train.py", line 88, in do_cli
[rank0]: return do_train(parsed_cfg, parsed_cli_args)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/axolotl/src/axolotl/cli/train.py", line 45, in do_train
[rank0]: model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper
[rank0]: return func(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/axolotl/src/axolotl/train.py", line 598, in train
[rank0]: execute_training(cfg, trainer, resume_from_checkpoint)
[rank0]: File "/workspace/axolotl/src/axolotl/train.py", line 213, in execute_training
[rank0]: trainer.train(resume_from_checkpoint=resume_from_checkpoint)
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2325, in train
[rank0]: return inner_training_loop(
[rank0]: ^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer.py", line 2573, in _inner_training_loop
[rank0]: self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 506, in on_train_begin
[rank0]: return self.call_event("on_train_begin", args, state, control)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/trainer_callback.py", line 556, in call_event
[rank0]: result = getattr(callback, event)(
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1489, in on_train_begin
[rank0]: self.setup(args, state, model)
[rank0]: File "/root/miniconda3/envs/py3.11/lib/python3.11/site-packages/transformers/integrations/integration_utils.py", line 1430, in setup
[rank0]: if not self._ml_flow.is_tracking_uri_set():
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: AttributeError: module 'mlflow' has no attribute 'is_tracking_uri_set'